talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into image-embedder-python

Browse Source
This commit is contained in:
Kinar R
2022-11-03 23:24:31 +05:30
committed by GitHub
parent 71d5b69544 b472d8ff66
commit 5a68ba84b6
244 changed files with 12284 additions and 2008 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/tasks/cc/components/BUILD
+2
View File
@@ -46,8 +46,10 @@ cc_library(
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/tasks/cc:common",
"//mediapipe/tasks/cc/core:model_resources",
"//mediapipe/tasks/cc/core/proto:acceleration_cc_proto",
"//mediapipe/tasks/cc/vision/utils:image_tensor_specs",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
mediapipe/tasks/cc/components/calculators/BUILD
+24
View File
@@ -44,6 +44,30 @@ cc_library(
alwayslink = 1,
)
cc_test(
name = "classification_aggregation_calculator_test",
srcs = ["classification_aggregation_calculator_test.cc"],
deps = [
":classification_aggregation_calculator",
":classification_aggregation_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:output_stream_poller",
"//mediapipe/framework:packet",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/api2:builder",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:str_format",
"@org_tensorflow//tensorflow/lite/core/shims:cc_shims_test_util",
],
)
mediapipe_proto_library(
name = "score_calibration_calculator_proto",
srcs = ["score_calibration_calculator.proto"],
mediapipe/tasks/cc/components/calculators/classification_aggregation_calculator.cc
+112 -19
View File
@@ -31,37 +31,62 @@
namespace mediapipe {
namespace api2 {
using ::mediapipe::tasks::ClassificationAggregationCalculatorOptions;
using ::mediapipe::tasks::components::containers::proto::ClassificationResult;
using ::mediapipe::tasks::components::containers::proto::Classifications;
// Aggregates ClassificationLists into a single ClassificationResult that has
// 3 dimensions: (classification head, classification timestamp, classification
// category).
// Aggregates ClassificationLists into either a ClassificationResult object
// representing the classification results aggregated by classifier head, or
// into an std::vector<ClassificationResult> representing the classification
// results aggregated first by timestamp then by classifier head.
//
// Inputs:
// CLASSIFICATIONS - ClassificationList
// CLASSIFICATIONS - ClassificationList @Multiple
// ClassificationList per classification head.
// TIMESTAMPS - std::vector<Timestamp> @Optional
// The collection of the timestamps that a single ClassificationResult
// should aggragate. This stream is optional, and the timestamp information
// will only be populated to the ClassificationResult proto when this stream
// is connected.
// The collection of the timestamps that this calculator should aggregate.
// This stream is optional: if provided then the TIMESTAMPED_CLASSIFICATIONS
// output is used for results. Otherwise as no timestamp aggregation is
// required the CLASSIFICATIONS output is used for results.
//
// Outputs:
// CLASSIFICATION_RESULT - ClassificationResult
// CLASSIFICATIONS - ClassificationResult @Optional
// The classification results aggregated by head. Must be connected if the
// TIMESTAMPS input is not connected, as it signals that timestamp
// aggregation is not required.
// TIMESTAMPED_CLASSIFICATIONS - std::vector<ClassificationResult> @Optional
// The classification result aggregated by timestamp, then by head. Must be
// connected if the TIMESTAMPS input is connected, as it signals that
// timestamp aggregation is required.
// // TODO: remove output once migration is over.
// CLASSIFICATION_RESULT - (DEPRECATED) ClassificationResult @Optional
// The aggregated classification result.
//
// Example:
// Example without timestamp aggregation:
// node {
// calculator: "ClassificationAggregationCalculator"
// input_stream: "CLASSIFICATIONS:0:stream_a"
// input_stream: "CLASSIFICATIONS:1:stream_b"
// input_stream: "CLASSIFICATIONS:2:stream_c"
// output_stream: "CLASSIFICATIONS:classifications"
// options {
// [mediapipe.ClassificationAggregationCalculatorOptions.ext] {
// head_names: "head_name_a"
// head_names: "head_name_b"
// head_names: "head_name_c"
// }
// }
// }
//
// Example with timestamp aggregation:
// node {
// calculator: "ClassificationAggregationCalculator"
// input_stream: "CLASSIFICATIONS:0:stream_a"
// input_stream: "CLASSIFICATIONS:1:stream_b"
// input_stream: "CLASSIFICATIONS:2:stream_c"
// input_stream: "TIMESTAMPS:timestamps"
// output_stream: "CLASSIFICATION_RESULT:classification_result"
// output_stream: "TIMESTAMPED_CLASSIFICATIONS:timestamped_classifications"
// options {
// [mediapipe.tasks.ClassificationAggregationCalculatorOptions.ext] {
// [mediapipe.ClassificationAggregationCalculatorOptions.ext] {
// head_names: "head_name_a"
// head_names: "head_name_b"
// head_names: "head_name_c"
@@ -74,8 +99,15 @@ class ClassificationAggregationCalculator : public Node {
"CLASSIFICATIONS"};
static constexpr Input<std::vector<Timestamp>>::Optional kTimestampsIn{
"TIMESTAMPS"};
static constexpr Output<ClassificationResult> kOut{"CLASSIFICATION_RESULT"};
MEDIAPIPE_NODE_CONTRACT(kClassificationListIn, kTimestampsIn, kOut);
static constexpr Output<ClassificationResult>::Optional kClassificationsOut{
"CLASSIFICATIONS"};
static constexpr Output<std::vector<ClassificationResult>>::Optional
kTimestampedClassificationsOut{"TIMESTAMPED_CLASSIFICATIONS"};
static constexpr Output<ClassificationResult>::Optional
kClassificationResultOut{"CLASSIFICATION_RESULT"};
MEDIAPIPE_NODE_CONTRACT(kClassificationListIn, kTimestampsIn,
kClassificationsOut, kTimestampedClassificationsOut,
kClassificationResultOut);
static absl::Status UpdateContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc);
@@ -88,6 +120,11 @@ class ClassificationAggregationCalculator : public Node {
cached_classifications_;
ClassificationResult ConvertToClassificationResult(CalculatorContext* cc);
std::vector<ClassificationResult> ConvertToTimestampedClassificationResults(
CalculatorContext* cc);
// TODO: deprecate this function once migration is over.
ClassificationResult LegacyConvertToClassificationResult(
CalculatorContext* cc);
};
absl::Status ClassificationAggregationCalculator::UpdateContract(
@@ -100,6 +137,10 @@ absl::Status ClassificationAggregationCalculator::UpdateContract(
<< "The size of classifications input streams should match the "
"size of head names specified in the calculator options";
}
// TODO: enforce connecting TIMESTAMPED_CLASSIFICATIONS if
// TIMESTAMPS is connected, and connecting CLASSIFICATIONS if TIMESTAMPS is
// not connected. All dependent tasks must be updated to use these outputs
// first.
return absl::OkStatus();
}
@@ -124,10 +165,19 @@ absl::Status ClassificationAggregationCalculator::Process(
[](const auto& elem) -> ClassificationList { return elem.Get(); });
cached_classifications_[cc->InputTimestamp().Value()] =
std::move(classification_lists);
if (time_aggregation_enabled_ && kTimestampsIn(cc).IsEmpty()) {
return absl::OkStatus();
ClassificationResult classification_result;
if (time_aggregation_enabled_) {
if (kTimestampsIn(cc).IsEmpty()) {
return absl::OkStatus();
}
classification_result = LegacyConvertToClassificationResult(cc);
kTimestampedClassificationsOut(cc).Send(
ConvertToTimestampedClassificationResults(cc));
} else {
classification_result = LegacyConvertToClassificationResult(cc);
kClassificationsOut(cc).Send(ConvertToClassificationResult(cc));
}
kOut(cc).Send(ConvertToClassificationResult(cc));
kClassificationResultOut(cc).Send(classification_result);
RET_CHECK(cached_classifications_.empty());
return absl::OkStatus();
}
@@ -136,6 +186,50 @@ ClassificationResult
ClassificationAggregationCalculator::ConvertToClassificationResult(
CalculatorContext* cc) {
ClassificationResult result;
auto& classification_lists =
cached_classifications_[cc->InputTimestamp().Value()];
for (int i = 0; i < classification_lists.size(); ++i) {
auto classifications = result.add_classifications();
classifications->set_head_index(i);
if (!head_names_.empty()) {
classifications->set_head_name(head_names_[i]);
}
*classifications->mutable_classification_list() =
std::move(classification_lists[i]);
}
cached_classifications_.erase(cc->InputTimestamp().Value());
return result;
}
std::vector<ClassificationResult>
ClassificationAggregationCalculator::ConvertToTimestampedClassificationResults(
CalculatorContext* cc) {
auto timestamps = kTimestampsIn(cc).Get();
std::vector<ClassificationResult> results;
results.reserve(timestamps.size());
for (const auto& timestamp : timestamps) {
ClassificationResult result;
result.set_timestamp_ms((timestamp.Value() - timestamps[0].Value()) / 1000);
auto& classification_lists = cached_classifications_[timestamp.Value()];
for (int i = 0; i < classification_lists.size(); ++i) {
auto classifications = result.add_classifications();
classifications->set_head_index(i);
if (!head_names_.empty()) {
classifications->set_head_name(head_names_[i]);
}
*classifications->mutable_classification_list() =
std::move(classification_lists[i]);
}
cached_classifications_.erase(timestamp.Value());
results.push_back(std::move(result));
}
return results;
}
ClassificationResult
ClassificationAggregationCalculator::LegacyConvertToClassificationResult(
CalculatorContext* cc) {
ClassificationResult result;
Timestamp first_timestamp(0);
std::vector<Timestamp> timestamps;
if (time_aggregation_enabled_) {
@@ -177,7 +271,6 @@ ClassificationAggregationCalculator::ConvertToClassificationResult(
entry->set_timestamp_ms((timestamp.Value() - first_timestamp.Value()) /
1000);
}
cached_classifications_.erase(timestamp.Value());
}
return result;
}
mediapipe/tasks/cc/components/calculators/classification_aggregation_calculator.proto
+1 -1
View File
@@ -15,7 +15,7 @@ limitations under the License.
syntax = "proto2";
package mediapipe.tasks;
package mediapipe;
import "mediapipe/framework/calculator.proto";
mediapipe/tasks/cc/components/calculators/classification_aggregation_calculator_test.cc
+213
View File
@@ -0,0 +1,213 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <memory>
#include <vector>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/output_stream_poller.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/tasks/cc/components/calculators/classification_aggregation_calculator.pb.h"
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
#include "tensorflow/lite/core/shims/cc/shims_test_util.h"
namespace mediapipe {
namespace {
using ::mediapipe::ParseTextProtoOrDie;
using ::mediapipe::api2::Input;
using ::mediapipe::api2::Output;
using ::mediapipe::api2::builder::Graph;
using ::mediapipe::api2::builder::Source;
using ::mediapipe::tasks::components::containers::proto::ClassificationResult;
using ::testing::Pointwise;
constexpr char kClassificationInput0Tag[] = "CLASSIFICATIONS_0";
constexpr char kClassificationInput0Name[] = "classifications_0";
constexpr char kClassificationInput1Tag[] = "CLASSIFICATIONS_1";
constexpr char kClassificationInput1Name[] = "classifications_1";
constexpr char kTimestampsTag[] = "TIMESTAMPS";
constexpr char kTimestampsName[] = "timestamps";
constexpr char kClassificationsTag[] = "CLASSIFICATIONS";
constexpr char kClassificationsName[] = "classifications";
constexpr char kTimestampedClassificationsTag[] = "TIMESTAMPED_CLASSIFICATIONS";
constexpr char kTimestampedClassificationsName[] =
"timestamped_classifications";
ClassificationList MakeClassificationList(int class_index) {
return ParseTextProtoOrDie<ClassificationList>(absl::StrFormat(
R"pb(
classification { index: %d }
)pb",
class_index));
}
class ClassificationAggregationCalculatorTest
: public tflite_shims::testing::Test {
protected:
absl::StatusOr<OutputStreamPoller> BuildGraph(
bool connect_timestamps = false) {
Graph graph;
auto& calculator = graph.AddNode("ClassificationAggregationCalculator");
calculator
.GetOptions<mediapipe::ClassificationAggregationCalculatorOptions>() =
ParseTextProtoOrDie<
mediapipe::ClassificationAggregationCalculatorOptions>(
R"pb(head_names: "foo" head_names: "bar")pb");
graph[Input<ClassificationList>(kClassificationInput0Tag)].SetName(
kClassificationInput0Name) >>
calculator.In(absl::StrFormat("%s:%d", kClassificationsTag, 0));
graph[Input<ClassificationList>(kClassificationInput1Tag)].SetName(
kClassificationInput1Name) >>
calculator.In(absl::StrFormat("%s:%d", kClassificationsTag, 1));
if (connect_timestamps) {
graph[Input<std::vector<Timestamp>>(kTimestampsTag)].SetName(
kTimestampsName) >>
calculator.In(kTimestampsTag);
calculator.Out(kTimestampedClassificationsTag)
.SetName(kTimestampedClassificationsName) >>
graph[Output<std::vector<ClassificationResult>>(
kTimestampedClassificationsTag)];
} else {
calculator.Out(kClassificationsTag).SetName(kClassificationsName) >>
graph[Output<ClassificationResult>(kClassificationsTag)];
}
MP_RETURN_IF_ERROR(calculator_graph_.Initialize(graph.GetConfig()));
if (connect_timestamps) {
ASSIGN_OR_RETURN(auto poller, calculator_graph_.AddOutputStreamPoller(
kTimestampedClassificationsName));
MP_RETURN_IF_ERROR(calculator_graph_.StartRun(/*extra_side_packets=*/{}));
return poller;
}
ASSIGN_OR_RETURN(auto poller, calculator_graph_.AddOutputStreamPoller(
kClassificationsName));
MP_RETURN_IF_ERROR(calculator_graph_.StartRun(/*extra_side_packets=*/{}));
return poller;
}
absl::Status Send(
std::vector<ClassificationList> classifications, int timestamp = 0,
std::optional<std::vector<int>> aggregation_timestamps = std::nullopt) {
MP_RETURN_IF_ERROR(calculator_graph_.AddPacketToInputStream(
kClassificationInput0Name,
MakePacket<ClassificationList>(classifications[0])
.At(Timestamp(timestamp))));
MP_RETURN_IF_ERROR(calculator_graph_.AddPacketToInputStream(
kClassificationInput1Name,
MakePacket<ClassificationList>(classifications[1])
.At(Timestamp(timestamp))));
if (aggregation_timestamps.has_value()) {
auto packet = std::make_unique<std::vector<Timestamp>>();
for (const auto& timestamp : *aggregation_timestamps) {
packet->emplace_back(Timestamp(timestamp));
}
MP_RETURN_IF_ERROR(calculator_graph_.AddPacketToInputStream(
kTimestampsName, Adopt(packet.release()).At(Timestamp(timestamp))));
}
return absl::OkStatus();
}
template <typename T>
absl::StatusOr<T> GetResult(OutputStreamPoller& poller) {
MP_RETURN_IF_ERROR(calculator_graph_.WaitUntilIdle());
MP_RETURN_IF_ERROR(calculator_graph_.CloseAllInputStreams());
Packet packet;
if (!poller.Next(&packet)) {
return absl::InternalError("Unable to get output packet");
}
auto result = packet.Get<T>();
MP_RETURN_IF_ERROR(calculator_graph_.WaitUntilDone());
return result;
}
private:
CalculatorGraph calculator_graph_;
};
TEST_F(ClassificationAggregationCalculatorTest, SucceedsWithoutTimestamps) {
MP_ASSERT_OK_AND_ASSIGN(auto poller, BuildGraph());
MP_ASSERT_OK(Send({MakeClassificationList(0), MakeClassificationList(1)}));
MP_ASSERT_OK_AND_ASSIGN(auto result, GetResult<ClassificationResult>(poller));
EXPECT_THAT(result,
EqualsProto(ParseTextProtoOrDie<ClassificationResult>(
R"pb(classifications {
head_index: 0
head_name: "foo"
classification_list { classification { index: 0 } }
}
classifications {
head_index: 1
head_name: "bar"
classification_list { classification { index: 1 } }
})pb")));
}
TEST_F(ClassificationAggregationCalculatorTest, SucceedsWithTimestamps) {
MP_ASSERT_OK_AND_ASSIGN(auto poller, BuildGraph(/*connect_timestamps=*/true));
MP_ASSERT_OK(Send({MakeClassificationList(0), MakeClassificationList(1)}));
MP_ASSERT_OK(Send(
{MakeClassificationList(2), MakeClassificationList(3)},
/*timestamp=*/1000,
/*aggregation_timestamps=*/std::optional<std::vector<int>>({0, 1000})));
MP_ASSERT_OK_AND_ASSIGN(auto result,
GetResult<std::vector<ClassificationResult>>(poller));
EXPECT_THAT(result,
Pointwise(EqualsProto(),
{ParseTextProtoOrDie<ClassificationResult>(R"pb(
timestamp_ms: 0,
classifications {
head_index: 0
head_name: "foo"
classification_list { classification { index: 0 } }
}
classifications {
head_index: 1
head_name: "bar"
classification_list { classification { index: 1 } }
}
)pb"),
ParseTextProtoOrDie<ClassificationResult>(R"pb(
timestamp_ms: 1,
classifications {
head_index: 0
head_name: "foo"
classification_list { classification { index: 2 } }
}
classifications {
head_index: 1
head_name: "bar"
classification_list { classification { index: 3 } }
}
)pb")}));
}
} // namespace
} // namespace mediapipe
mediapipe/tasks/cc/components/containers/BUILD
+20
View File
@@ -29,3 +29,23 @@ cc_library(
"//mediapipe/framework/formats:landmark_cc_proto",
],
)
cc_library(
name = "category",
srcs = ["category.cc"],
hdrs = ["category.h"],
deps = [
"//mediapipe/framework/formats:classification_cc_proto",
],
)
cc_library(
name = "classification_result",
srcs = ["classification_result.cc"],
hdrs = ["classification_result.h"],
deps = [
":category",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto",
],
)
mediapipe/tasks/cc/components/containers/category.cc
+38
View File
@@ -0,0 +1,38 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/cc/components/containers/category.h"
#include <optional>
#include <string>
#include "mediapipe/framework/formats/classification.pb.h"
namespace mediapipe::tasks::components::containers {
Category ConvertToCategory(const mediapipe::Classification& proto) {
Category category;
category.index = proto.index();
category.score = proto.score();
if (proto.has_label()) {
category.category_name = proto.label();
}
if (proto.has_display_name()) {
category.display_name = proto.display_name();
}
return category;
}
} // namespace mediapipe::tasks::components::containers
mediapipe/tasks/cc/components/containers/category.h
+52
View File
@@ -0,0 +1,52 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_CATEGORY_H_
#define MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_CATEGORY_H_
#include <optional>
#include <string>
#include "mediapipe/framework/formats/classification.pb.h"
namespace mediapipe::tasks::components::containers {
// Defines a single classification result.
//
// The label maps packed into the TFLite Model Metadata [1] are used to populate
// the 'category_name' and 'display_name' fields.
//
// [1]: https://www.tensorflow.org/lite/convert/metadata
struct Category {
// The index of the category in the classification model output.
int index;
// The score for this category, e.g. (but not necessarily) a probability in
// [0,1].
float score;
// The optional ID for the category, read from the label map packed in the
// TFLite Model Metadata if present. Not necessarily human-readable.
std::optional<std::string> category_name = std::nullopt;
// The optional human-readable name for the category, read from the label map
// packed in the TFLite Model Metadata if present.
std::optional<std::string> display_name = std::nullopt;
};
// Utility function to convert from mediapipe::Classification proto to Category
// struct.
Category ConvertToCategory(const mediapipe::Classification& proto);
} // namespace mediapipe::tasks::components::containers
#endif // MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_CATEGORY_H_
mediapipe/tasks/cc/components/containers/classification_result.cc
+57
View File
@@ -0,0 +1,57 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/cc/components/containers/classification_result.h"
#include <optional>
#include <string>
#include <vector>
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/tasks/cc/components/containers/category.h"
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
namespace mediapipe::tasks::components::containers {
Classifications ConvertToClassifications(const proto::Classifications& proto) {
Classifications classifications;
classifications.categories.reserve(
proto.classification_list().classification_size());
for (const auto& classification :
proto.classification_list().classification()) {
classifications.categories.push_back(ConvertToCategory(classification));
}
classifications.head_index = proto.head_index();
if (proto.has_head_name()) {
classifications.head_name = proto.head_name();
}
return classifications;
}
ClassificationResult ConvertToClassificationResult(
const proto::ClassificationResult& proto) {
ClassificationResult classification_result;
classification_result.classifications.reserve(proto.classifications_size());
for (const auto& classifications : proto.classifications()) {
classification_result.classifications.push_back(
ConvertToClassifications(classifications));
}
if (proto.has_timestamp_ms()) {
classification_result.timestamp_ms = proto.timestamp_ms();
}
return classification_result;
}
} // namespace mediapipe::tasks::components::containers
mediapipe/tasks/cc/components/containers/classification_result.h
+68
View File
@@ -0,0 +1,68 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_
#define MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_
#include <optional>
#include <string>
#include <vector>
#include "mediapipe/tasks/cc/components/containers/category.h"
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
namespace mediapipe::tasks::components::containers {
// Defines classification results for a given classifier head.
struct Classifications {
// The array of predicted categories, usually sorted by descending scores,
// e.g. from high to low probability.
std::vector<Category> categories;
// The index of the classifier head (i.e. output tensor) these categories
// refer to. This is useful for multi-head models.
int head_index;
// The optional name of the classifier head, as provided in the TFLite Model
// Metadata [1] if present. This is useful for multi-head models.
//
// [1]: https://www.tensorflow.org/lite/convert/metadata
std::optional<std::string> head_name = std::nullopt;
};
// Defines classification results of a model.
struct ClassificationResult {
// The classification results for each head of the model.
std::vector<Classifications> classifications;
// The optional timestamp (in milliseconds) of the start of the chunk of data
// corresponding to these results.
//
// This is only used for classification on time series (e.g. audio
// classification). In these use cases, the amount of data to process might
// exceed the maximum size that the model can process: to solve this, the
// input data is split into multiple chunks starting at different timestamps.
std::optional<int64_t> timestamp_ms = std::nullopt;
};
// Utility function to convert from Classifications proto to
// Classifications struct.
Classifications ConvertToClassifications(const proto::Classifications& proto);
// Utility function to convert from ClassificationResult proto to
// ClassificationResult struct.
ClassificationResult ConvertToClassificationResult(
const proto::ClassificationResult& proto);
} // namespace mediapipe::tasks::components::containers
#endif // MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_
mediapipe/tasks/cc/components/containers/proto/BUILD
+1
View File
@@ -28,6 +28,7 @@ mediapipe_proto_library(
srcs = ["classifications.proto"],
deps = [
":category_proto",
"//mediapipe/framework/formats:classification_proto",
],
)
mediapipe/tasks/cc/components/containers/proto/category.proto
+2 -1
View File
@@ -17,9 +17,10 @@ syntax = "proto2";
package mediapipe.tasks.components.containers.proto;
option java_package = "com.google.mediapipe.tasks.components.container.proto";
option java_package = "com.google.mediapipe.tasks.components.containers.proto";
option java_outer_classname = "CategoryProto";
// TODO: deprecate this message once migration is over.
// A single classification result.
message Category {
// The index of the category in the corresponding label map, usually packed in
mediapipe/tasks/cc/components/containers/proto/classifications.proto
+18 -3
View File
@@ -17,11 +17,13 @@ syntax = "proto2";
package mediapipe.tasks.components.containers.proto;
import "mediapipe/framework/formats/classification.proto";
import "mediapipe/tasks/cc/components/containers/proto/category.proto";
option java_package = "com.google.mediapipe.tasks.components.container.proto";
option java_package = "com.google.mediapipe.tasks.components.containers.proto";
option java_outer_classname = "ClassificationsProto";
// TODO: deprecate this message once migration is over.
// List of predicted categories with an optional timestamp.
message ClassificationEntry {
// The array of predicted categories, usually sorted by descending scores,
@@ -33,9 +35,12 @@ message ClassificationEntry {
optional int64 timestamp_ms = 2;
}
// Classifications for a given classifier head.
// Classifications for a given classifier head, i.e. for a given output tensor.
message Classifications {
// TODO: deprecate this field once migration is over.
repeated ClassificationEntry entries = 1;
// The classification results for this head.
optional mediapipe.ClassificationList classification_list = 4;
// The index of the classifier head these categories refer to. This is useful
// for multi-head models.
optional int32 head_index = 2;
@@ -45,7 +50,17 @@ message Classifications {
optional string head_name = 3;
}
// Contains one set of results per classifier head.
// Classifications for a given classifier model.
message ClassificationResult {
// The classification results for each model head, i.e. one for each output
// tensor.
repeated Classifications classifications = 1;
// The optional timestamp (in milliseconds) of the start of the chunk of data
// corresponding to these results.
//
// This is only used for classification on time series (e.g. audio
// classification). In these use cases, the amount of data to process might
// exceed the maximum size that the model can process: to solve this, the
// input data is split into multiple chunks starting at different timestamps.
optional int64 timestamp_ms = 2;
}
mediapipe/tasks/cc/components/containers/proto/embeddings.proto
+3
View File
@@ -17,6 +17,9 @@ syntax = "proto2";
package mediapipe.tasks.components.containers.proto;
option java_package = "com.google.mediapipe.tasks.components.containers.proto";
option java_outer_classname = "EmbeddingsProto";
// Defines a dense floating-point embedding.
message FloatEmbedding {
repeated float values = 1 [packed = true];
mediapipe/tasks/cc/components/image_preprocessing.cc
+14 -1
View File
@@ -30,9 +30,11 @@ limitations under the License.
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/gpu/gpu_origin.pb.h"
#include "mediapipe/tasks/cc/common.h"
#include "mediapipe/tasks/cc/components/image_preprocessing_options.pb.h"
#include "mediapipe/tasks/cc/core/model_resources.h"
#include "mediapipe/tasks/cc/core/proto/acceleration.pb.h"
#include "mediapipe/tasks/cc/vision/utils/image_tensor_specs.h"
#include "tensorflow/lite/schema/schema_generated.h"
@@ -128,12 +130,21 @@ absl::Status ConfigureImageToTensorCalculator(
options->mutable_output_tensor_float_range()->set_max((255.0f - mean) /
std);
}
// TODO: need to support different GPU origin on differnt
// platforms or applications.
options->set_gpu_origin(mediapipe::GpuOrigin::TOP_LEFT);
return absl::OkStatus();
}
} // namespace
bool DetermineImagePreprocessingGpuBackend(
const core::proto::Acceleration& acceleration) {
return acceleration.has_gpu();
}
absl::Status ConfigureImagePreprocessing(const ModelResources& model_resources,
bool use_gpu,
ImagePreprocessingOptions* options) {
ASSIGN_OR_RETURN(auto image_tensor_specs,
BuildImageTensorSpecs(model_resources));
@@ -141,7 +152,9 @@ absl::Status ConfigureImagePreprocessing(const ModelResources& model_resources,
image_tensor_specs, options->mutable_image_to_tensor_options()));
// The GPU backend isn't able to process int data. If the input tensor is
// quantized, forces the image preprocessing graph to use CPU backend.
if (image_tensor_specs.tensor_type == tflite::TensorType_UINT8) {
if (use_gpu && image_tensor_specs.tensor_type != tflite::TensorType_UINT8) {
options->set_backend(ImagePreprocessingOptions::GPU_BACKEND);
} else {
options->set_backend(ImagePreprocessingOptions::CPU_BACKEND);
}
return absl::OkStatus();
mediapipe/tasks/cc/components/image_preprocessing.h
+13 -2
View File
@@ -19,20 +19,26 @@ limitations under the License.
#include "absl/status/status.h"
#include "mediapipe/tasks/cc/components/image_preprocessing_options.pb.h"
#include "mediapipe/tasks/cc/core/model_resources.h"
#include "mediapipe/tasks/cc/core/proto/acceleration.pb.h"
namespace mediapipe {
namespace tasks {
namespace components {
// Configures an ImagePreprocessing subgraph using the provided model resources.
// Configures an ImagePreprocessing subgraph using the provided model resources
// When use_gpu is true, use GPU as backend to convert image to tensor.
// - Accepts CPU input images and outputs CPU tensors.
//
// Example usage:
//
// auto& preprocessing =
// graph.AddNode("mediapipe.tasks.components.ImagePreprocessingSubgraph");
// core::proto::Acceleration acceleration;
// acceleration.mutable_xnnpack();
// bool use_gpu = DetermineImagePreprocessingGpuBackend(acceleration);
// MP_RETURN_IF_ERROR(ConfigureImagePreprocessing(
// model_resources,
// use_gpu,
// &preprocessing.GetOptions<ImagePreprocessingOptions>()));
//
// The resulting ImagePreprocessing subgraph has the following I/O:
@@ -56,9 +62,14 @@ namespace components {
// The image that has the pixel data stored on the target storage (CPU vs
// GPU).
absl::Status ConfigureImagePreprocessing(
const core::ModelResources& model_resources,
const core::ModelResources& model_resources, bool use_gpu,
ImagePreprocessingOptions* options);
// Determine if the image preprocessing subgraph should use GPU as the backend
// according to the given acceleration setting.
bool DetermineImagePreprocessingGpuBackend(
const core::proto::Acceleration& acceleration);
} // namespace components
} // namespace tasks
} // namespace mediapipe
mediapipe/tasks/cc/components/processors/classification_postprocessing_graph.cc
+53 -15
View File
@@ -78,6 +78,14 @@ constexpr char kClassificationsTag[] = "CLASSIFICATIONS";
constexpr char kScoresTag[] = "SCORES";
constexpr char kTensorsTag[] = "TENSORS";
constexpr char kTimestampsTag[] = "TIMESTAMPS";
constexpr char kTimestampedClassificationsTag[] = "TIMESTAMPED_CLASSIFICATIONS";
// Struct holding the different output streams produced by the graph.
struct ClassificationPostprocessingOutputStreams {
Source<ClassificationResult> classification_result;
Source<ClassificationResult> classifications;
Source<std::vector<ClassificationResult>> timestamped_classifications;
};
// Performs sanity checks on provided ClassifierOptions.
absl::Status SanityCheckClassifierOptions(
@@ -286,7 +294,7 @@ absl::Status ConfigureScoreCalibrationIfAny(
void ConfigureClassificationAggregationCalculator(
const ModelMetadataExtractor& metadata_extractor,
ClassificationAggregationCalculatorOptions* options) {
mediapipe::ClassificationAggregationCalculatorOptions* options) {
auto* output_tensors_metadata = metadata_extractor.GetOutputTensorMetadata();
if (output_tensors_metadata == nullptr) {
return;
@@ -378,12 +386,23 @@ absl::Status ConfigureClassificationPostprocessingGraph(
// TENSORS - std::vector<Tensor>
// The output tensors of an InferenceCalculator.
// TIMESTAMPS - std::vector<Timestamp> @Optional
// The collection of timestamps that a single ClassificationResult should
// aggregate. This is mostly useful for classifiers working on time series,
// e.g. audio or video classification.
// The collection of the timestamps that this calculator should aggregate.
// This stream is optional: if provided then the TIMESTAMPED_CLASSIFICATIONS
// output is used for results. Otherwise as no timestamp aggregation is
// required the CLASSIFICATIONS output is used for results.
//
// Outputs:
// CLASSIFICATION_RESULT - ClassificationResult
// The output aggregated classification results.
// CLASSIFICATIONS - ClassificationResult @Optional
// The classification results aggregated by head. Must be connected if the
// TIMESTAMPS input is not connected, as it signals that timestamp
// aggregation is not required.
// TIMESTAMPED_CLASSIFICATIONS - std::vector<ClassificationResult> @Optional
// The classification result aggregated by timestamp, then by head. Must be
// connected if the TIMESTAMPS input is connected, as it signals that
// timestamp aggregation is required.
// // TODO: remove output once migration is over.
// CLASSIFICATION_RESULT - (DEPRECATED) ClassificationResult @Optional
// The aggregated classification result.
//
// The recommended way of using this graph is through the GraphBuilder API
// using the 'ConfigureClassificationPostprocessingGraph()' function. See header
@@ -394,28 +413,39 @@ class ClassificationPostprocessingGraph : public mediapipe::Subgraph {
mediapipe::SubgraphContext* sc) override {
Graph graph;
ASSIGN_OR_RETURN(
auto classification_result_out,
auto output_streams,
BuildClassificationPostprocessing(
sc->Options<proto::ClassificationPostprocessingGraphOptions>(),
graph[Input<std::vector<Tensor>>(kTensorsTag)],
graph[Input<std::vector<Timestamp>>(kTimestampsTag)], graph));
classification_result_out >>
output_streams.classification_result >>
graph[Output<ClassificationResult>(kClassificationResultTag)];
output_streams.classifications >>
graph[Output<ClassificationResult>(kClassificationsTag)];
output_streams.timestamped_classifications >>
graph[Output<std::vector<ClassificationResult>>(
kTimestampedClassificationsTag)];
return graph.GetConfig();
}
private:
// Adds an on-device classification postprocessing graph into the provided
// builder::Graph instance. The classification postprocessing graph takes
// tensors (std::vector<mediapipe::Tensor>) as input and returns one output
// stream containing the output classification results (ClassificationResult).
// tensors (std::vector<mediapipe::Tensor>) and optional timestamps
// (std::vector<Timestamp>) as input and returns two output streams:
// - classification results aggregated by classifier head as a
// ClassificationResult proto, used when no timestamps are passed in
// the graph,
// - classification results aggregated by timestamp then by classifier head
// as a std::vector<ClassificationResult>, used when timestamps are passed
// in the graph.
//
// options: the on-device ClassificationPostprocessingGraphOptions.
// tensors_in: (std::vector<mediapipe::Tensor>>) tensors to postprocess.
// timestamps_in: (std::vector<mediapipe::Timestamp>) optional collection of
// timestamps that a single ClassificationResult should aggregate.
// timestamps that should be used to aggregate classification results.
// graph: the mediapipe builder::Graph instance to be updated.
absl::StatusOr<Source<ClassificationResult>>
absl::StatusOr<ClassificationPostprocessingOutputStreams>
BuildClassificationPostprocessing(
const proto::ClassificationPostprocessingGraphOptions& options,
Source<std::vector<Tensor>> tensors_in,
@@ -494,7 +524,8 @@ class ClassificationPostprocessingGraph : public mediapipe::Subgraph {
// Aggregates Classifications into a single ClassificationResult.
auto& result_aggregation =
graph.AddNode("ClassificationAggregationCalculator");
result_aggregation.GetOptions<ClassificationAggregationCalculatorOptions>()
result_aggregation
.GetOptions<mediapipe::ClassificationAggregationCalculatorOptions>()
.CopyFrom(options.classification_aggregation_options());
for (int i = 0; i < num_heads; ++i) {
tensors_to_classification_nodes[i]->Out(kClassificationsTag) >>
@@ -504,8 +535,15 @@ class ClassificationPostprocessingGraph : public mediapipe::Subgraph {
timestamps_in >> result_aggregation.In(kTimestampsTag);
// Connects output.
return result_aggregation[Output<ClassificationResult>(
kClassificationResultTag)];
ClassificationPostprocessingOutputStreams output_streams{
/*classification_result=*/result_aggregation
[Output<ClassificationResult>(kClassificationResultTag)],
/*classifications=*/
result_aggregation[Output<ClassificationResult>(kClassificationsTag)],
/*timestamped_classifications=*/
result_aggregation[Output<std::vector<ClassificationResult>>(
kTimestampedClassificationsTag)]};
return output_streams;
}
};
mediapipe/tasks/cc/components/processors/classification_postprocessing_graph.h
+15 -5
View File
@@ -45,12 +45,22 @@ namespace processors {
// TENSORS - std::vector<Tensor>
// The output tensors of an InferenceCalculator.
// TIMESTAMPS - std::vector<Timestamp> @Optional
// The collection of timestamps that a single ClassificationResult should
// aggregate. This is mostly useful for classifiers working on time series,
// e.g. audio or video classification.
// The collection of the timestamps that this calculator should aggregate.
// This stream is optional: if provided then the TIMESTAMPED_CLASSIFICATIONS
// output is used for results. Otherwise as no timestamp aggregation is
// required the CLASSIFICATIONS output is used for results.
// Outputs:
// CLASSIFICATION_RESULT - ClassificationResult
// The output aggregated classification results.
// CLASSIFICATIONS - ClassificationResult @Optional
// The classification results aggregated by head. Must be connected if the
// TIMESTAMPS input is not connected, as it signals that timestamp
// aggregation is not required.
// TIMESTAMPED_CLASSIFICATIONS - std::vector<ClassificationResult> @Optional
// The classification result aggregated by timestamp, then by head. Must be
// connected if the TIMESTAMPS input is connected, as it signals that
// timestamp aggregation is required.
// // TODO: remove output once migration is over.
// CLASSIFICATION_RESULT - (DEPRECATED) ClassificationResult @Optional
// The aggregated classification result.
absl::Status ConfigureClassificationPostprocessingGraph(
const tasks::core::ModelResources& model_resources,
const proto::ClassifierOptions& classifier_options,
mediapipe/tasks/cc/components/processors/classification_postprocessing_graph_test.cc
+322 -5
View File
@@ -38,6 +38,7 @@ limitations under the License.
#include "mediapipe/framework/output_stream_poller.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/tasks/cc/components/calculators/classification_aggregation_calculator.pb.h"
@@ -64,6 +65,7 @@ using ::mediapipe::file::JoinPath;
using ::mediapipe::tasks::components::containers::proto::ClassificationResult;
using ::mediapipe::tasks::core::ModelResources;
using ::testing::HasSubstr;
using ::testing::Pointwise;
using ::testing::proto::Approximately;
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/";
@@ -86,6 +88,11 @@ constexpr char kTimestampsTag[] = "TIMESTAMPS";
constexpr char kTimestampsName[] = "timestamps";
constexpr char kClassificationResultTag[] = "CLASSIFICATION_RESULT";
constexpr char kClassificationResultName[] = "classification_result";
constexpr char kClassificationsTag[] = "CLASSIFICATIONS";
constexpr char kClassificationsName[] = "classifications";
constexpr char kTimestampedClassificationsTag[] = "TIMESTAMPED_CLASSIFICATIONS";
constexpr char kTimestampedClassificationsName[] =
"timestamped_classifications";
// Helper function to get ModelResources.
absl::StatusOr<std::unique_ptr<ModelResources>> CreateModelResourcesForModel(
@@ -413,6 +420,316 @@ TEST_F(ConfigureTest, SucceedsWithMultipleHeads) {
}
class PostprocessingTest : public tflite_shims::testing::Test {
protected:
absl::StatusOr<OutputStreamPoller> BuildGraph(
absl::string_view model_name, const proto::ClassifierOptions& options,
bool connect_timestamps = false) {
ASSIGN_OR_RETURN(auto model_resources,
CreateModelResourcesForModel(model_name));
Graph graph;
auto& postprocessing = graph.AddNode(
"mediapipe.tasks.components.processors."
"ClassificationPostprocessingGraph");
MP_RETURN_IF_ERROR(ConfigureClassificationPostprocessingGraph(
*model_resources, options,
&postprocessing
.GetOptions<proto::ClassificationPostprocessingGraphOptions>()));
graph[Input<std::vector<Tensor>>(kTensorsTag)].SetName(kTensorsName) >>
postprocessing.In(kTensorsTag);
if (connect_timestamps) {
graph[Input<std::vector<Timestamp>>(kTimestampsTag)].SetName(
kTimestampsName) >>
postprocessing.In(kTimestampsTag);
postprocessing.Out(kTimestampedClassificationsTag)
.SetName(kTimestampedClassificationsName) >>
graph[Output<std::vector<ClassificationResult>>(
kTimestampedClassificationsTag)];
} else {
postprocessing.Out(kClassificationsTag).SetName(kClassificationsName) >>
graph[Output<ClassificationResult>(kClassificationsTag)];
}
MP_RETURN_IF_ERROR(calculator_graph_.Initialize(graph.GetConfig()));
if (connect_timestamps) {
ASSIGN_OR_RETURN(auto poller, calculator_graph_.AddOutputStreamPoller(
kTimestampedClassificationsName));
MP_RETURN_IF_ERROR(calculator_graph_.StartRun(/*extra_side_packets=*/{}));
return poller;
}
ASSIGN_OR_RETURN(auto poller, calculator_graph_.AddOutputStreamPoller(
kClassificationsName));
MP_RETURN_IF_ERROR(calculator_graph_.StartRun(/*extra_side_packets=*/{}));
return poller;
}
template <typename T>
void AddTensor(
const std::vector<T>& tensor, const Tensor::ElementType& element_type,
const Tensor::QuantizationParameters& quantization_parameters = {}) {
tensors_->emplace_back(element_type,
Tensor::Shape{1, static_cast<int>(tensor.size())},
quantization_parameters);
auto view = tensors_->back().GetCpuWriteView();
T* buffer = view.buffer<T>();
std::copy(tensor.begin(), tensor.end(), buffer);
}
absl::Status Run(
std::optional<std::vector<int>> aggregation_timestamps = std::nullopt,
int timestamp = 0) {
MP_RETURN_IF_ERROR(calculator_graph_.AddPacketToInputStream(
kTensorsName, Adopt(tensors_.release()).At(Timestamp(timestamp))));
// Reset tensors for future calls.
tensors_ = absl::make_unique<std::vector<Tensor>>();
if (aggregation_timestamps.has_value()) {
auto packet = absl::make_unique<std::vector<Timestamp>>();
for (const auto& timestamp : *aggregation_timestamps) {
packet->emplace_back(Timestamp(timestamp));
}
MP_RETURN_IF_ERROR(calculator_graph_.AddPacketToInputStream(
kTimestampsName, Adopt(packet.release()).At(Timestamp(timestamp))));
}
return absl::OkStatus();
}
template <typename T>
absl::StatusOr<T> GetResult(OutputStreamPoller& poller) {
MP_RETURN_IF_ERROR(calculator_graph_.WaitUntilIdle());
MP_RETURN_IF_ERROR(calculator_graph_.CloseAllInputStreams());
Packet packet;
if (!poller.Next(&packet)) {
return absl::InternalError("Unable to get output packet");
}
auto result = packet.Get<T>();
MP_RETURN_IF_ERROR(calculator_graph_.WaitUntilDone());
return result;
}
private:
CalculatorGraph calculator_graph_;
std::unique_ptr<std::vector<Tensor>> tensors_ =
absl::make_unique<std::vector<Tensor>>();
};
TEST_F(PostprocessingTest, SucceedsWithoutMetadata) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(3);
options.set_score_threshold(0.5);
MP_ASSERT_OK_AND_ASSIGN(
auto poller,
BuildGraph(kQuantizedImageClassifierWithoutMetadata, options));
// Build input tensors.
std::vector<uint8> tensor(kMobileNetNumClasses, 0);
tensor[1] = 18;
tensor[2] = 16;
// Send tensors and get results.
AddTensor(tensor, Tensor::ElementType::kUInt8,
/*quantization_parameters=*/{0.1, 10});
MP_ASSERT_OK(Run());
MP_ASSERT_OK_AND_ASSIGN(auto results,
GetResult<ClassificationResult>(poller));
// Validate results.
EXPECT_THAT(results,
EqualsProto(ParseTextProtoOrDie<ClassificationResult>(R"pb(
classifications {
head_index: 0
classification_list {
classification { index: 1 score: 0.8 }
classification { index: 2 score: 0.6 }
}
}
)pb")));
}
TEST_F(PostprocessingTest, SucceedsWithMetadata) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(3);
MP_ASSERT_OK_AND_ASSIGN(
auto poller, BuildGraph(kQuantizedImageClassifierWithMetadata, options));
// Build input tensors.
std::vector<uint8> tensor(kMobileNetNumClasses, 0);
tensor[1] = 12;
tensor[2] = 14;
tensor[3] = 16;
tensor[4] = 18;
// Send tensors and get results.
AddTensor(tensor, Tensor::ElementType::kUInt8,
/*quantization_parameters=*/{0.1, 10});
MP_ASSERT_OK(Run());
MP_ASSERT_OK_AND_ASSIGN(auto results,
GetResult<ClassificationResult>(poller));
// Validate results.
EXPECT_THAT(
results, EqualsProto(ParseTextProtoOrDie<ClassificationResult>(R"pb(
classifications {
head_index: 0
head_name: "probability"
classification_list {
classification { index: 4 score: 0.8 label: "tiger shark" }
classification { index: 3 score: 0.6 label: "great white shark" }
classification { index: 2 score: 0.4 label: "goldfish" }
}
}
)pb")));
}
TEST_F(PostprocessingTest, SucceedsWithScoreCalibration) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(3);
MP_ASSERT_OK_AND_ASSIGN(
auto poller,
BuildGraph(kQuantizedImageClassifierWithDummyScoreCalibration, options));
// Build input tensors.
std::vector<uint8> tensor(kMobileNetNumClasses, 0);
tensor[1] = 12;
tensor[2] = 14;
tensor[3] = 16;
tensor[4] = 18;
// Send tensors and get results.
AddTensor(tensor, Tensor::ElementType::kUInt8,
/*quantization_parameters=*/{0.1, 10});
MP_ASSERT_OK(Run());
MP_ASSERT_OK_AND_ASSIGN(auto results,
GetResult<ClassificationResult>(poller));
// Validate results.
EXPECT_THAT(
results, EqualsProto(ParseTextProtoOrDie<ClassificationResult>(R"pb(
classifications {
head_index: 0
head_name: "probability"
classification_list {
classification { index: 4 score: 0.6899744811 label: "tiger shark" }
classification {
index: 3
score: 0.6456563062
label: "great white shark"
}
classification { index: 2 score: 0.5986876601 label: "goldfish" }
}
}
)pb")));
}
TEST_F(PostprocessingTest, SucceedsWithMultipleHeads) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(2);
MP_ASSERT_OK_AND_ASSIGN(
auto poller,
BuildGraph(kFloatTwoHeadsAudioClassifierWithMetadata, options));
// Build input tensors.
std::vector<float> tensor_0(kTwoHeadsNumClasses[0], 0);
tensor_0[1] = 0.2;
tensor_0[2] = 0.4;
tensor_0[3] = 0.6;
std::vector<float> tensor_1(kTwoHeadsNumClasses[1], 0);
tensor_1[1] = 0.2;
tensor_1[2] = 0.4;
tensor_1[3] = 0.6;
// Send tensors and get results.
AddTensor(tensor_0, Tensor::ElementType::kFloat32);
AddTensor(tensor_1, Tensor::ElementType::kFloat32);
MP_ASSERT_OK(Run());
MP_ASSERT_OK_AND_ASSIGN(auto results,
GetResult<ClassificationResult>(poller));
// Validate results.
EXPECT_THAT(
results, EqualsProto(ParseTextProtoOrDie<ClassificationResult>(R"pb(
classifications {
head_index: 0
head_name: "yamnet_classification"
classification_list {
classification { index: 3 score: 0.6 label: "Narration, monologue" }
classification { index: 2 score: 0.4 label: "Conversation" }
}
}
classifications {
head_index: 1
head_name: "bird_classification"
classification_list {
classification { index: 3 score: 0.6 label: "Azara\'s Spinetail" }
classification { index: 2 score: 0.4 label: "House Sparrow" }
}
}
)pb")));
}
TEST_F(PostprocessingTest, SucceedsWithTimestamps) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(2);
MP_ASSERT_OK_AND_ASSIGN(
auto poller, BuildGraph(kQuantizedImageClassifierWithMetadata, options,
/*connect_timestamps=*/true));
// Build input tensors.
std::vector<uint8> tensor_0(kMobileNetNumClasses, 0);
tensor_0[1] = 12;
tensor_0[2] = 14;
tensor_0[3] = 16;
std::vector<uint8> tensor_1(kMobileNetNumClasses, 0);
tensor_1[5] = 12;
tensor_1[6] = 14;
tensor_1[7] = 16;
// Send tensors and get results.
AddTensor(tensor_0, Tensor::ElementType::kUInt8,
/*quantization_parameters=*/{0.1, 10});
MP_ASSERT_OK(Run());
AddTensor(tensor_1, Tensor::ElementType::kUInt8,
/*quantization_parameters=*/{0.1, 10});
MP_ASSERT_OK(Run(
/*aggregation_timestamps=*/std::optional<std::vector<int>>({0, 1000}),
/*timestamp=*/1000));
MP_ASSERT_OK_AND_ASSIGN(auto results,
GetResult<std::vector<ClassificationResult>>(poller));
// Validate results.
EXPECT_THAT(
results,
Pointwise(
EqualsProto(),
{ParseTextProtoOrDie<ClassificationResult>(R"pb(
timestamp_ms: 0
classifications {
head_index: 0
head_name: "probability"
classification_list {
classification {
index: 3
score: 0.6
label: "great white shark"
}
classification { index: 2 score: 0.4 label: "goldfish" }
}
})pb"),
ParseTextProtoOrDie<ClassificationResult>(R"pb(
timestamp_ms: 1
classifications {
head_index: 0
head_name: "probability"
classification_list {
classification { index: 7 score: 0.6 label: "stingray" }
classification { index: 6 score: 0.4 label: "electric ray" }
}
})pb")}));
}
// TODO: remove these tests once migration is over.
class LegacyPostprocessingTest : public tflite_shims::testing::Test {
protected:
absl::StatusOr<OutputStreamPoller> BuildGraph(
absl::string_view model_name, const proto::ClassifierOptions& options,
@@ -496,7 +813,7 @@ class PostprocessingTest : public tflite_shims::testing::Test {
absl::make_unique<std::vector<Tensor>>();
};
TEST_F(PostprocessingTest, SucceedsWithoutMetadata) {
TEST_F(LegacyPostprocessingTest, SucceedsWithoutMetadata) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(3);
@@ -525,7 +842,7 @@ TEST_F(PostprocessingTest, SucceedsWithoutMetadata) {
})pb"));
}
TEST_F(PostprocessingTest, SucceedsWithMetadata) {
TEST_F(LegacyPostprocessingTest, SucceedsWithMetadata) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(3);
@@ -568,7 +885,7 @@ TEST_F(PostprocessingTest, SucceedsWithMetadata) {
})pb"));
}
TEST_F(PostprocessingTest, SucceedsWithScoreCalibration) {
TEST_F(LegacyPostprocessingTest, SucceedsWithScoreCalibration) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(3);
@@ -614,7 +931,7 @@ TEST_F(PostprocessingTest, SucceedsWithScoreCalibration) {
})pb"));
}
TEST_F(PostprocessingTest, SucceedsWithMultipleHeads) {
TEST_F(LegacyPostprocessingTest, SucceedsWithMultipleHeads) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(2);
@@ -674,7 +991,7 @@ TEST_F(PostprocessingTest, SucceedsWithMultipleHeads) {
})pb"));
}
TEST_F(PostprocessingTest, SucceedsWithTimestamps) {
TEST_F(LegacyPostprocessingTest, SucceedsWithTimestamps) {
// Build graph.
proto::ClassifierOptions options;
options.set_max_results(2);
mediapipe/tasks/cc/components/processors/proto/classification_postprocessing_graph_options.proto
+1 -1
View File
@@ -38,7 +38,7 @@ message ClassificationPostprocessingGraphOptions {
// Options for the ClassificationAggregationCalculator encapsulated by the
// ClassificationPostprocessing subgraph.
optional ClassificationAggregationCalculatorOptions
optional mediapipe.ClassificationAggregationCalculatorOptions
classification_aggregation_options = 2;
// Whether output tensors are quantized (kTfLiteUint8) or not (kFloat32).
mediapipe/tasks/cc/core/model_task_graph.cc
+15 -9
View File
@@ -156,21 +156,24 @@ absl::StatusOr<CalculatorGraphConfig> ModelTaskGraph::GetConfig(
}
absl::StatusOr<const ModelResources*> ModelTaskGraph::CreateModelResources(
SubgraphContext* sc, std::unique_ptr<proto::ExternalFile> external_file) {
SubgraphContext* sc, std::unique_ptr<proto::ExternalFile> external_file,
const std::string tag_suffix) {
auto model_resources_cache_service = sc->Service(kModelResourcesCacheService);
if (!model_resources_cache_service.IsAvailable()) {
ASSIGN_OR_RETURN(local_model_resources_,
ASSIGN_OR_RETURN(auto local_model_resource,
ModelResources::Create("", std::move(external_file)));
LOG(WARNING)
<< "A local ModelResources object is created. Please consider using "
"ModelResourcesCacheService to cache the created ModelResources "
"object in the CalculatorGraph.";
return local_model_resources_.get();
local_model_resources_.push_back(std::move(local_model_resource));
return local_model_resources_.back().get();
}
ASSIGN_OR_RETURN(
auto op_resolver_packet,
model_resources_cache_service.GetObject().GetGraphOpResolverPacket());
const std::string tag = CreateModelResourcesTag(sc->OriginalNode());
const std::string tag =
absl::StrCat(CreateModelResourcesTag(sc->OriginalNode()), tag_suffix);
ASSIGN_OR_RETURN(auto model_resources,
ModelResources::Create(tag, std::move(external_file),
op_resolver_packet));
@@ -182,7 +185,8 @@ absl::StatusOr<const ModelResources*> ModelTaskGraph::CreateModelResources(
absl::StatusOr<const ModelAssetBundleResources*>
ModelTaskGraph::CreateModelAssetBundleResources(
SubgraphContext* sc, std::unique_ptr<proto::ExternalFile> external_file) {
SubgraphContext* sc, std::unique_ptr<proto::ExternalFile> external_file,
const std::string tag_suffix) {
auto model_resources_cache_service = sc->Service(kModelResourcesCacheService);
bool has_file_pointer_meta = external_file->has_file_pointer_meta();
// if external file is set by file pointer, no need to add the model asset
@@ -190,7 +194,7 @@ ModelTaskGraph::CreateModelAssetBundleResources(
// not owned by this model asset bundle resources.
if (!model_resources_cache_service.IsAvailable() || has_file_pointer_meta) {
ASSIGN_OR_RETURN(
local_model_asset_bundle_resources_,
auto local_model_asset_bundle_resource,
ModelAssetBundleResources::Create("", std::move(external_file)));
if (!has_file_pointer_meta) {
LOG(WARNING)
@@ -198,10 +202,12 @@ ModelTaskGraph::CreateModelAssetBundleResources(
"ModelResourcesCacheService to cache the created ModelResources "
"object in the CalculatorGraph.";
}
return local_model_asset_bundle_resources_.get();
local_model_asset_bundle_resources_.push_back(
std::move(local_model_asset_bundle_resource));
return local_model_asset_bundle_resources_.back().get();
}
const std::string tag =
CreateModelAssetBundleResourcesTag(sc->OriginalNode());
const std::string tag = absl::StrCat(
CreateModelAssetBundleResourcesTag(sc->OriginalNode()), tag_suffix);
ASSIGN_OR_RETURN(
auto model_bundle_resources,
ModelAssetBundleResources::Create(tag, std::move(external_file)));
mediapipe/tasks/cc/core/model_task_graph.h
+17 -6
View File
@@ -19,6 +19,7 @@ limitations under the License.
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
@@ -75,9 +76,14 @@ class ModelTaskGraph : public Subgraph {
// construction stage. Note that the external file contents will be moved
// into the model resources object on creation. The returned model resources
// pointer will provide graph authors with the access to the metadata
// extractor and the tflite model.
// extractor and the tflite model. When the model resources graph service is
// available, a tag is generated internally asscoiated with the created model
// resource. If more than one model resources are created in a graph, the
// model resources graph service add the tag_suffix to support multiple
// resources.
absl::StatusOr<const ModelResources*> CreateModelResources(
SubgraphContext* sc, std::unique_ptr<proto::ExternalFile> external_file);
SubgraphContext* sc, std::unique_ptr<proto::ExternalFile> external_file,
const std::string tag_suffix = "");
// If the model resources graph service is available, creates a model asset
// bundle resources object from the subgraph context, and caches the created
@@ -103,10 +109,15 @@ class ModelTaskGraph : public Subgraph {
// that can only be used in the graph construction stage. Note that the
// external file contents will be moved into the model asset bundle resources
// object on creation. The returned model asset bundle resources pointer will
// provide graph authors with the access to extracted model files.
// provide graph authors with the access to extracted model files. When the
// model resources graph service is available, a tag is generated internally
// asscoiated with the created model asset bundle resource. If more than one
// model asset bundle resources are created in a graph, the model resources
// graph service add the tag_suffix to support multiple resources.
absl::StatusOr<const ModelAssetBundleResources*>
CreateModelAssetBundleResources(
SubgraphContext* sc, std::unique_ptr<proto::ExternalFile> external_file);
SubgraphContext* sc, std::unique_ptr<proto::ExternalFile> external_file,
const std::string tag_suffix = "");
// Inserts a mediapipe task inference subgraph into the provided
// GraphBuilder. The returned node provides the following interfaces to the
@@ -124,9 +135,9 @@ class ModelTaskGraph : public Subgraph {
api2::builder::Graph& graph) const;
private:
std::unique_ptr<ModelResources> local_model_resources_;
std::vector<std::unique_ptr<ModelResources>> local_model_resources_;
std::unique_ptr<ModelAssetBundleResources>
std::vector<std::unique_ptr<ModelAssetBundleResources>>
local_model_asset_bundle_resources_;
};
mediapipe/tasks/cc/text/text_classifier/BUILD
+26
View File
@@ -49,6 +49,8 @@ cc_library(
":text_classifier_graph",
"//mediapipe/framework:packet",
"//mediapipe/framework/api2:builder",
"//mediapipe/tasks/cc/components/containers:category",
"//mediapipe/tasks/cc/components/containers:classification_result",
"//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto",
"//mediapipe/tasks/cc/components/processors:classifier_options",
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto",
@@ -63,6 +65,30 @@ cc_library(
],
)
cc_test(
name = "text_classifier_test",
srcs = ["text_classifier_test.cc"],
data = [
"//mediapipe/tasks/testdata/text:bert_text_classifier_models",
"//mediapipe/tasks/testdata/text:text_classifier_models",
],
deps = [
":text_classifier",
":text_classifier_test_utils",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/tasks/cc:common",
"//mediapipe/tasks/cc/components/containers:category",
"//mediapipe/tasks/cc/components/containers:classification_result",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:cord",
"@org_tensorflow//tensorflow/lite/core/shims:cc_shims_test_util",
],
)
cc_library(
name = "text_classifier_test_utils",
srcs = ["text_classifier_test_utils.cc"],
mediapipe/tasks/cc/text/text_classifier/text_classifier.cc
+9 -8
View File
@@ -24,6 +24,7 @@ limitations under the License.
#include "absl/strings/string_view.h"
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/tasks/cc/components/containers/classification_result.h"
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h"
#include "mediapipe/tasks/cc/core/task_api_factory.h"
@@ -37,12 +38,13 @@ namespace text_classifier {
namespace {
using ::mediapipe::tasks::components::containers::ConvertToClassificationResult;
using ::mediapipe::tasks::components::containers::proto::ClassificationResult;
constexpr char kTextStreamName[] = "text_in";
constexpr char kTextTag[] = "TEXT";
constexpr char kClassificationResultStreamName[] = "classification_result_out";
constexpr char kClassificationResultTag[] = "CLASSIFICATION_RESULT";
constexpr char kClassificationsStreamName[] = "classifications_out";
constexpr char kClassificationsTag[] = "CLASSIFICATIONS";
constexpr char kSubgraphTypeName[] =
"mediapipe.tasks.text.text_classifier.TextClassifierGraph";
@@ -54,9 +56,8 @@ CalculatorGraphConfig CreateGraphConfig(
auto& subgraph = graph.AddNode(kSubgraphTypeName);
subgraph.GetOptions<proto::TextClassifierGraphOptions>().Swap(options.get());
graph.In(kTextTag).SetName(kTextStreamName) >> subgraph.In(kTextTag);
subgraph.Out(kClassificationResultTag)
.SetName(kClassificationResultStreamName) >>
graph.Out(kClassificationResultTag);
subgraph.Out(kClassificationsTag).SetName(kClassificationsStreamName) >>
graph.Out(kClassificationsTag);
return graph.GetConfig();
}
@@ -88,14 +89,14 @@ absl::StatusOr<std::unique_ptr<TextClassifier>> TextClassifier::Create(
std::move(options->base_options.op_resolver));
}
absl::StatusOr<ClassificationResult> TextClassifier::Classify(
absl::StatusOr<TextClassifierResult> TextClassifier::Classify(
absl::string_view text) {
ASSIGN_OR_RETURN(
auto output_packets,
runner_->Process(
{{kTextStreamName, MakePacket<std::string>(std::string(text))}}));
return output_packets[kClassificationResultStreamName]
.Get<ClassificationResult>();
return ConvertToClassificationResult(
output_packets[kClassificationsStreamName].Get<ClassificationResult>());
}
} // namespace text_classifier
mediapipe/tasks/cc/text/text_classifier/text_classifier.h
+6 -3
View File
@@ -21,7 +21,7 @@ limitations under the License.
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
#include "mediapipe/tasks/cc/components/containers/classification_result.h"
#include "mediapipe/tasks/cc/components/processors/classifier_options.h"
#include "mediapipe/tasks/cc/core/base_options.h"
#include "mediapipe/tasks/cc/core/base_task_api.h"
@@ -31,6 +31,10 @@ namespace tasks {
namespace text {
namespace text_classifier {
// Alias the shared ClassificationResult struct as result type.
using TextClassifierResult =
::mediapipe::tasks::components::containers::ClassificationResult;
// The options for configuring a MediaPipe text classifier task.
struct TextClassifierOptions {
// Base options for configuring MediaPipe Tasks, such as specifying the model
@@ -81,8 +85,7 @@ class TextClassifier : core::BaseTaskApi {
std::unique_ptr<TextClassifierOptions> options);
// Performs classification on the input `text`.
absl::StatusOr<components::containers::proto::ClassificationResult> Classify(
absl::string_view text);
absl::StatusOr<TextClassifierResult> Classify(absl::string_view text);
// Shuts down the TextClassifier when all the work is done.
absl::Status Close() { return runner_->Close(); }
mediapipe/tasks/cc/text/text_classifier/text_classifier_graph.cc
+23 -7
View File
@@ -47,10 +47,18 @@ using ::mediapipe::tasks::components::containers::proto::ClassificationResult;
using ::mediapipe::tasks::core::ModelResources;
constexpr char kClassificationResultTag[] = "CLASSIFICATION_RESULT";
constexpr char kClassificationsTag[] = "CLASSIFICATIONS";
constexpr char kTextTag[] = "TEXT";
constexpr char kMetadataExtractorTag[] = "METADATA_EXTRACTOR";
constexpr char kTensorsTag[] = "TENSORS";
// TODO: remove once Java API migration is over.
// Struct holding the different output streams produced by the text classifier.
struct TextClassifierOutputStreams {
Source<ClassificationResult> classification_result;
Source<ClassificationResult> classifications;
};
} // namespace
// A "TextClassifierGraph" performs Natural Language classification (including
@@ -62,7 +70,10 @@ constexpr char kTensorsTag[] = "TENSORS";
// Input text to perform classification on.
//
// Outputs:
// CLASSIFICATION_RESULT - ClassificationResult
// CLASSIFICATIONS - ClassificationResult @Optional
// The classification results aggregated by classifier head.
// TODO: remove once Java API migration is over.
// CLASSIFICATION_RESULT - (DEPRECATED) ClassificationResult @Optional
// The aggregated classification result object that has 3 dimensions:
// (classification head, classification timestamp, classification category).
//
@@ -70,7 +81,7 @@ constexpr char kTensorsTag[] = "TENSORS";
// node {
// calculator: "mediapipe.tasks.text.text_classifier.TextClassifierGraph"
// input_stream: "TEXT:text_in"
// output_stream: "CLASSIFICATION_RESULT:classification_result_out"
// output_stream: "CLASSIFICATIONS:classifications_out"
// options {
// [mediapipe.tasks.text.text_classifier.proto.TextClassifierGraphOptions.ext]
// {
@@ -91,12 +102,14 @@ class TextClassifierGraph : public core::ModelTaskGraph {
CreateModelResources<proto::TextClassifierGraphOptions>(sc));
Graph graph;
ASSIGN_OR_RETURN(
Source<ClassificationResult> classification_result_out,
auto output_streams,
BuildTextClassifierTask(
sc->Options<proto::TextClassifierGraphOptions>(), *model_resources,
graph[Input<std::string>(kTextTag)], graph));
classification_result_out >>
output_streams.classification_result >>
graph[Output<ClassificationResult>(kClassificationResultTag)];
output_streams.classifications >>
graph[Output<ClassificationResult>(kClassificationsTag)];
return graph.GetConfig();
}
@@ -111,7 +124,7 @@ class TextClassifierGraph : public core::ModelTaskGraph {
// TextClassifier model file with model metadata.
// text_in: (std::string) stream to run text classification on.
// graph: the mediapipe builder::Graph instance to be updated.
absl::StatusOr<Source<ClassificationResult>> BuildTextClassifierTask(
absl::StatusOr<TextClassifierOutputStreams> BuildTextClassifierTask(
const proto::TextClassifierGraphOptions& task_options,
const ModelResources& model_resources, Source<std::string> text_in,
Graph& graph) {
@@ -148,8 +161,11 @@ class TextClassifierGraph : public core::ModelTaskGraph {
// Outputs the aggregated classification result as the subgraph output
// stream.
return postprocessing[Output<ClassificationResult>(
kClassificationResultTag)];
return TextClassifierOutputStreams{
/*classification_result=*/postprocessing[Output<ClassificationResult>(
kClassificationResultTag)],
/*classifications=*/postprocessing[Output<ClassificationResult>(
kClassificationsTag)]};
}
};
mediapipe/tasks/cc/text/text_classifier/text_classifier_test.cc
+78 -92
View File
@@ -33,7 +33,8 @@ limitations under the License.
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/tasks/cc/common.h"
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
#include "mediapipe/tasks/cc/components/containers/category.h"
#include "mediapipe/tasks/cc/components/containers/classification_result.h"
#include "mediapipe/tasks/cc/text/text_classifier/text_classifier_test_utils.h"
#include "tensorflow/lite/core/shims/cc/shims_test_util.h"
@@ -43,17 +44,13 @@ namespace text {
namespace text_classifier {
namespace {
using ::mediapipe::EqualsProto;
using ::mediapipe::file::JoinPath;
using ::mediapipe::tasks::kMediaPipeTasksPayload;
using ::mediapipe::tasks::components::containers::proto::ClassificationResult;
using ::mediapipe::tasks::components::containers::Category;
using ::mediapipe::tasks::components::containers::Classifications;
using ::testing::HasSubstr;
using ::testing::Optional;
using ::testing::proto::Approximately;
using ::testing::proto::IgnoringRepeatedFieldOrdering;
using ::testing::proto::Partially;
constexpr float kEpsilon = 0.001;
constexpr int kMaxSeqLen = 128;
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/text/";
constexpr char kTestBertModelPath[] = "bert_text_classifier.tflite";
@@ -67,6 +64,30 @@ std::string GetFullPath(absl::string_view file_name) {
return JoinPath("./", kTestDataDirectory, file_name);
}
// Checks that the two provided `TextClassifierResult` are equal, with a
// tolerancy on floating-point score to account for numerical instabilities.
// TODO: create shared matcher for ClassificationResult.
void ExpectApproximatelyEqual(const TextClassifierResult& actual,
const TextClassifierResult& expected) {
const float kPrecision = 1e-6;
ASSERT_EQ(actual.classifications.size(), expected.classifications.size());
for (int i = 0; i < actual.classifications.size(); ++i) {
const Classifications& a = actual.classifications[i];
const Classifications& b = expected.classifications[i];
EXPECT_EQ(a.head_index, b.head_index);
EXPECT_EQ(a.head_name, b.head_name);
EXPECT_EQ(a.categories.size(), b.categories.size());
for (int j = 0; j < a.categories.size(); ++j) {
const Category& x = a.categories[j];
const Category& y = b.categories[j];
EXPECT_EQ(x.index, y.index);
EXPECT_NEAR(x.score, y.score, kPrecision);
EXPECT_EQ(x.category_name, y.category_name);
EXPECT_EQ(x.display_name, y.display_name);
}
}
}
class TextClassifierTest : public tflite_shims::testing::Test {};
TEST_F(TextClassifierTest, CreateSucceedsWithBertModel) {
@@ -116,34 +137,29 @@ TEST_F(TextClassifierTest, TextClassifierWithBert) {
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<TextClassifier> classifier,
TextClassifier::Create(std::move(options)));
MP_ASSERT_OK_AND_ASSIGN(
ClassificationResult negative_result,
TextClassifierResult negative_result,
classifier->Classify("unflinchingly bleak and desperate"));
ASSERT_THAT(negative_result,
Partially(IgnoringRepeatedFieldOrdering(Approximately(
EqualsProto(R"pb(
classifications {
entries {
categories { category_name: "negative" score: 0.956 }
categories { category_name: "positive" score: 0.044 }
}
}
)pb"),
kEpsilon))));
TextClassifierResult negative_expected;
negative_expected.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/0, /*score=*/0.956317, /*category_name=*/"negative"},
{/*index=*/1, /*score=*/0.043683, /*category_name=*/"positive"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(negative_result, negative_expected);
MP_ASSERT_OK_AND_ASSIGN(
ClassificationResult positive_result,
TextClassifierResult positive_result,
classifier->Classify("it's a charming and often affecting journey"));
ASSERT_THAT(positive_result,
Partially(IgnoringRepeatedFieldOrdering(Approximately(
EqualsProto(R"pb(
classifications {
entries {
categories { category_name: "negative" score: 0.0 }
categories { category_name: "positive" score: 1.0 }
}
}
)pb"),
kEpsilon))));
TextClassifierResult positive_expected;
positive_expected.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/1, /*score=*/0.999945, /*category_name=*/"positive"},
{/*index=*/0, /*score=*/0.000056, /*category_name=*/"negative"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(positive_result, positive_expected);
MP_ASSERT_OK(classifier->Close());
}
@@ -152,35 +168,30 @@ TEST_F(TextClassifierTest, TextClassifierWithIntInputs) {
options->base_options.model_asset_path = GetFullPath(kTestRegexModelPath);
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<TextClassifier> classifier,
TextClassifier::Create(std::move(options)));
MP_ASSERT_OK_AND_ASSIGN(ClassificationResult negative_result,
MP_ASSERT_OK_AND_ASSIGN(TextClassifierResult negative_result,
classifier->Classify("What a waste of my time."));
ASSERT_THAT(negative_result,
Partially(IgnoringRepeatedFieldOrdering(Approximately(
EqualsProto(R"pb(
classifications {
entries {
categories { category_name: "Negative" score: 0.813 }
categories { category_name: "Positive" score: 0.187 }
}
}
)pb"),
kEpsilon))));
TextClassifierResult negative_expected;
negative_expected.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/0, /*score=*/0.813130, /*category_name=*/"Negative"},
{/*index=*/1, /*score=*/0.186870, /*category_name=*/"Positive"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(negative_result, negative_expected);
MP_ASSERT_OK_AND_ASSIGN(
ClassificationResult positive_result,
classifier->Classify("This is the best movie I’ve seen in recent years. "
TextClassifierResult positive_result,
classifier->Classify("This is the best movie I’ve seen in recent years."
"Strongly recommend it!"));
ASSERT_THAT(positive_result,
Partially(IgnoringRepeatedFieldOrdering(Approximately(
EqualsProto(R"pb(
classifications {
entries {
categories { category_name: "Negative" score: 0.487 }
categories { category_name: "Positive" score: 0.513 }
}
}
)pb"),
kEpsilon))));
TextClassifierResult positive_expected;
positive_expected.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/1, /*score=*/0.513427, /*category_name=*/"Positive"},
{/*index=*/0, /*score=*/0.486573, /*category_name=*/"Negative"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(positive_result, positive_expected);
MP_ASSERT_OK(classifier->Close());
}
@@ -190,44 +201,19 @@ TEST_F(TextClassifierTest, TextClassifierWithStringToBool) {
options->base_options.op_resolver = CreateCustomResolver();
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<TextClassifier> classifier,
TextClassifier::Create(std::move(options)));
MP_ASSERT_OK_AND_ASSIGN(ClassificationResult result,
MP_ASSERT_OK_AND_ASSIGN(TextClassifierResult result,
classifier->Classify("hello"));
ASSERT_THAT(result, Partially(IgnoringRepeatedFieldOrdering(EqualsProto(R"pb(
classifications {
entries {
categories { index: 1 score: 1 }
categories { index: 0 score: 1 }
categories { index: 2 score: 0 }
}
}
)pb"))));
}
TEST_F(TextClassifierTest, BertLongPositive) {
std::stringstream ss_for_positive_review;
ss_for_positive_review
<< "it's a charming and often affecting journey and this is a long";
for (int i = 0; i < kMaxSeqLen; ++i) {
ss_for_positive_review << " long";
}
ss_for_positive_review << " movie review";
auto options = std::make_unique<TextClassifierOptions>();
options->base_options.model_asset_path = GetFullPath(kTestBertModelPath);
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<TextClassifier> classifier,
TextClassifier::Create(std::move(options)));
MP_ASSERT_OK_AND_ASSIGN(ClassificationResult result,
classifier->Classify(ss_for_positive_review.str()));
ASSERT_THAT(result,
Partially(IgnoringRepeatedFieldOrdering(Approximately(
EqualsProto(R"pb(
classifications {
entries {
categories { category_name: "negative" score: 0.014 }
categories { category_name: "positive" score: 0.986 }
}
}
)pb"),
kEpsilon))));
// Binary outputs causes flaky ordering, so we compare manually.
ASSERT_EQ(result.classifications.size(), 1);
ASSERT_EQ(result.classifications[0].head_index, 0);
ASSERT_EQ(result.classifications[0].categories.size(), 3);
ASSERT_EQ(result.classifications[0].categories[0].score, 1);
ASSERT_LT(result.classifications[0].categories[0].index, 2); // i.e O or 1.
ASSERT_EQ(result.classifications[0].categories[1].score, 1);
ASSERT_LT(result.classifications[0].categories[1].index, 2); // i.e 0 or 1.
ASSERT_EQ(result.classifications[0].categories[2].score, 0);
ASSERT_EQ(result.classifications[0].categories[2].index, 2);
MP_ASSERT_OK(classifier->Close());
}
mediapipe/tasks/cc/text/tokenizers/BUILD
+38 -2
View File
@@ -73,7 +73,18 @@ cc_library(
],
)
# TODO: This test fails in OSS
cc_test(
name = "sentencepiece_tokenizer_test",
srcs = ["sentencepiece_tokenizer_test.cc"],
data = [
"//mediapipe/tasks/testdata/text:albert_model",
],
deps = [
":sentencepiece_tokenizer",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/tasks/cc/core:utils",
],
)
cc_library(
name = "tokenizer_utils",
@@ -97,7 +108,32 @@ cc_library(
],
)
# TODO: This test fails in OSS
cc_test(
name = "tokenizer_utils_test",
srcs = ["tokenizer_utils_test.cc"],
data = [
"//mediapipe/tasks/testdata/text:albert_model",
"//mediapipe/tasks/testdata/text:mobile_bert_model",
"//mediapipe/tasks/testdata/text:text_classifier_models",
],
linkopts = ["-ldl"],
deps = [
":bert_tokenizer",
":regex_tokenizer",
":sentencepiece_tokenizer",
":tokenizer_utils",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status",
"//mediapipe/tasks/cc:common",
"//mediapipe/tasks/cc/core:utils",
"//mediapipe/tasks/cc/metadata:metadata_extractor",
"//mediapipe/tasks/metadata:metadata_schema_cc",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:cord",
],
)
cc_library(
name = "regex_tokenizer",
mediapipe/tasks/cc/vision/core/BUILD
+11
View File
@@ -21,12 +21,23 @@ cc_library(
hdrs = ["running_mode.h"],
)
cc_library(
name = "image_processing_options",
hdrs = ["image_processing_options.h"],
deps = [
"//mediapipe/tasks/cc/components/containers:rect",
],
)
cc_library(
name = "base_vision_task_api",
hdrs = ["base_vision_task_api.h"],
deps = [
":image_processing_options",
":running_mode",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/tasks/cc/components/containers:rect",
"//mediapipe/tasks/cc/core:base_task_api",
"//mediapipe/tasks/cc/core:task_runner",
"@com_google_absl//absl/status",
mediapipe/tasks/cc/vision/core/base_vision_task_api.h
+59
View File
@@ -16,15 +16,20 @@ limitations under the License.
#ifndef MEDIAPIPE_TASKS_CC_VISION_CORE_BASE_VISION_TASK_API_H_
#define MEDIAPIPE_TASKS_CC_VISION_CORE_BASE_VISION_TASK_API_H_
#include <cmath>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/tasks/cc/components/containers/rect.h"
#include "mediapipe/tasks/cc/core/base_task_api.h"
#include "mediapipe/tasks/cc/core/task_runner.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
namespace mediapipe {
@@ -87,6 +92,60 @@ class BaseVisionTaskApi : public tasks::core::BaseTaskApi {
return runner_->Send(std::move(inputs));
}
// Convert from ImageProcessingOptions to NormalizedRect, performing sanity
// checks on-the-fly. If the input ImageProcessingOptions is not present,
// returns a default NormalizedRect covering the whole image with rotation set
// to 0. If 'roi_allowed' is false, an error will be returned if the input
// ImageProcessingOptions has its 'region_or_interest' field set.
static absl::StatusOr<mediapipe::NormalizedRect> ConvertToNormalizedRect(
std::optional<ImageProcessingOptions> options, bool roi_allowed = true) {
mediapipe::NormalizedRect normalized_rect;
normalized_rect.set_rotation(0);
normalized_rect.set_x_center(0.5);
normalized_rect.set_y_center(0.5);
normalized_rect.set_width(1.0);
normalized_rect.set_height(1.0);
if (!options.has_value()) {
return normalized_rect;
}
if (options->rotation_degrees % 90 != 0) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"Expected rotation to be a multiple of 90°.",
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError);
}
// Convert to radians counter-clockwise.
normalized_rect.set_rotation(-options->rotation_degrees * M_PI / 180.0);
if (options->region_of_interest.has_value()) {
if (!roi_allowed) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"This task doesn't support region-of-interest.",
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError);
}
auto& roi = *options->region_of_interest;
if (roi.left >= roi.right || roi.top >= roi.bottom) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"Expected Rect with left < right and top < bottom.",
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError);
}
if (roi.left < 0 || roi.top < 0 || roi.right > 1 || roi.bottom > 1) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"Expected Rect values to be in [0,1].",
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError);
}
normalized_rect.set_x_center((roi.left + roi.right) / 2.0);
normalized_rect.set_y_center((roi.top + roi.bottom) / 2.0);
normalized_rect.set_width(roi.right - roi.left);
normalized_rect.set_height(roi.bottom - roi.top);
}
return normalized_rect;
}
private:
RunningMode running_mode_;
};
mediapipe/tasks/cc/vision/core/image_processing_options.h
+52
View File
@@ -0,0 +1,52 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef MEDIAPIPE_TASKS_CC_VISION_CORE_IMAGE_PROCESSING_OPTIONS_H_
#define MEDIAPIPE_TASKS_CC_VISION_CORE_IMAGE_PROCESSING_OPTIONS_H_
#include <optional>
#include "mediapipe/tasks/cc/components/containers/rect.h"
namespace mediapipe {
namespace tasks {
namespace vision {
namespace core {
// Options for image processing.
//
// If both region-or-interest and rotation are specified, the crop around the
// region-of-interest is extracted first, the the specified rotation is applied
// to the crop.
struct ImageProcessingOptions {
// The optional region-of-interest to crop from the image. If not specified,
// the full image is used.
//
// Coordinates must be in [0,1] with 'left' < 'right' and 'top' < bottom.
std::optional<components::containers::Rect> region_of_interest = std::nullopt;
// The rotation to apply to the image (or cropped region-of-interest), in
// degrees clockwise.
//
// The rotation must be a multiple (positive or negative) of 90°.
int rotation_degrees = 0;
};
} // namespace core
} // namespace vision
} // namespace tasks
} // namespace mediapipe
#endif // MEDIAPIPE_TASKS_CC_VISION_CORE_IMAGE_PROCESSING_OPTIONS_H_
mediapipe/tasks/cc/vision/gesture_recognizer/BUILD
+20 -2
View File
@@ -56,22 +56,31 @@ cc_library(
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/tasks/cc:common",
"//mediapipe/tasks/cc/components:image_preprocessing",
"//mediapipe/tasks/cc/components/processors:classification_postprocessing_graph",
"//mediapipe/tasks/cc/components/processors/proto:classification_postprocessing_graph_options_cc_proto",
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto",
"//mediapipe/tasks/cc/core:model_asset_bundle_resources",
"//mediapipe/tasks/cc/core:model_resources",
"//mediapipe/tasks/cc/core:model_resources_cache",
"//mediapipe/tasks/cc/core:model_task_graph",
"//mediapipe/tasks/cc/core:utils",
"//mediapipe/tasks/cc/core/proto:base_options_cc_proto",
"//mediapipe/tasks/cc/core/proto:external_file_cc_proto",
"//mediapipe/tasks/cc/core/proto:inference_subgraph_cc_proto",
"//mediapipe/tasks/cc/metadata/utils:zip_utils",
"//mediapipe/tasks/cc/vision/gesture_recognizer/calculators:combined_prediction_calculator",
"//mediapipe/tasks/cc/vision/gesture_recognizer/calculators:combined_prediction_calculator_cc_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/calculators:handedness_to_matrix_calculator",
"//mediapipe/tasks/cc/vision/gesture_recognizer/calculators:landmarks_to_matrix_calculator",
"//mediapipe/tasks/cc/vision/gesture_recognizer/calculators:landmarks_to_matrix_calculator_cc_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_embedder_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/hand_landmarker:hand_landmarks_detector_graph",
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_cc_proto",
"//mediapipe/tasks/metadata:metadata_schema_cc",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
@@ -91,10 +100,15 @@ cc_library(
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:status",
"//mediapipe/tasks/cc:common",
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto",
"//mediapipe/tasks/cc/core:model_asset_bundle_resources",
"//mediapipe/tasks/cc/core:model_resources_cache",
"//mediapipe/tasks/cc/core:model_task_graph",
"//mediapipe/tasks/cc/core:utils",
"//mediapipe/tasks/cc/metadata/utils:zip_utils",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/hand_detector:hand_detector_graph",
@@ -123,9 +137,11 @@ cc_library(
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/tasks/cc:common",
"//mediapipe/tasks/cc/components:image_preprocessing",
"//mediapipe/tasks/cc/components/containers:gesture_recognition_result",
"//mediapipe/tasks/cc/components/processors:classifier_options",
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto",
"//mediapipe/tasks/cc/core:base_options",
"//mediapipe/tasks/cc/core:base_task_api",
@@ -134,8 +150,10 @@ cc_library(
"//mediapipe/tasks/cc/core:utils",
"//mediapipe/tasks/cc/core/proto:inference_subgraph_cc_proto",
"//mediapipe/tasks/cc/vision/core:base_vision_task_api",
"//mediapipe/tasks/cc/vision/core:image_processing_options",
"//mediapipe/tasks/cc/vision/core:running_mode",
"//mediapipe/tasks/cc/vision/core:vision_task_api_factory",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_cc_proto",
mediapipe/tasks/cc/vision/gesture_recognizer/calculators/BUILD
+45
View File
@@ -69,6 +69,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
@@ -86,8 +87,52 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/strings",
],
)
mediapipe_proto_library(
name = "combined_prediction_calculator_proto",
srcs = ["combined_prediction_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
name = "combined_prediction_calculator",
srcs = ["combined_prediction_calculator.cc"],
deps = [
":combined_prediction_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:collection",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/api2:packet",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:classification_cc_proto",
"@com_google_absl//absl/container:btree",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:str_format",
],
alwayslink = 1,
)
cc_test(
name = "combined_prediction_calculator_test",
srcs = ["combined_prediction_calculator_test.cc"],
deps = [
":combined_prediction_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/port:gtest",
"//mediapipe/framework/port:gtest_main",
"@com_google_absl//absl/strings",
],
)
mediapipe/tasks/cc/vision/gesture_recognizer/calculators/combined_prediction_calculator.cc
+186
View File
@@ -0,0 +1,186 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/container/btree_map.h"
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/api2/packet.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/collection.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/calculators/combined_prediction_calculator.pb.h"
namespace mediapipe {
namespace api2 {
namespace {
constexpr char kPredictionTag[] = "PREDICTION";
Classification GetMaxScoringClassification(
const ClassificationList& classifications) {
Classification max_classification;
max_classification.set_score(0);
for (const auto& input : classifications.classification()) {
if (max_classification.score() < input.score()) {
max_classification = input;
}
}
return max_classification;
}
float GetScoreThreshold(
const std::string& input_label,
const absl::btree_map<std::string, float>& classwise_thresholds,
const std::string& background_label, const float default_threshold) {
float threshold = default_threshold;
auto it = classwise_thresholds.find(input_label);
if (it != classwise_thresholds.end()) {
threshold = it->second;
}
return threshold;
}
std::unique_ptr<ClassificationList> GetWinningPrediction(
const ClassificationList& classification_list,
const absl::btree_map<std::string, float>& classwise_thresholds,
const std::string& background_label, const float default_threshold) {
auto prediction_list = std::make_unique<ClassificationList>();
if (classification_list.classification().empty()) {
return prediction_list;
}
Classification& prediction = *prediction_list->add_classification();
auto argmax_prediction = GetMaxScoringClassification(classification_list);
float argmax_prediction_thresh =
GetScoreThreshold(argmax_prediction.label(), classwise_thresholds,
background_label, default_threshold);
if (argmax_prediction.score() >= argmax_prediction_thresh) {
prediction.set_label(argmax_prediction.label());
prediction.set_score(argmax_prediction.score());
} else {
for (const auto& input : classification_list.classification()) {
if (input.label() == background_label) {
prediction.set_label(input.label());
prediction.set_score(input.score());
break;
}
}
}
return prediction_list;
}
} // namespace
// This calculator accepts multiple ClassificationList input streams. Each
// ClassificationList should contain classifications with labels and
// corresponding softmax scores. The calculator computes the best prediction for
// each ClassificationList input stream via argmax and thresholding. Thresholds
// for all classes can be specified in the
// `CombinedPredictionCalculatorOptions`, along with a default global
// threshold.
// Please note that for this calculator to work as designed, the class names
// other than the background class in the ClassificationList objects must be
// different, but the background class name has to be the same. This background
// label name can be set via `background_label` in
// `CombinedPredictionCalculatorOptions`.
// The ClassificationList in the PREDICTION output stream contains the label of
// the winning class and corresponding softmax score. If none of the
// ClassificationList objects has a non-background winning class, the output
// contains the background class and score of the background class in the first
// ClassificationList. If multiple ClassificationList objects have a
// non-background winning class, the output contains the winning prediction from
// the ClassificationList with the highest priority. Priority is in decreasing
// order of input streams to the graph node using this calculator.
// Input:
// At least one stream with ClassificationList.
// Output:
// PREDICTION - A ClassificationList with the winning label as the only item.
//
// Usage example:
// node {
// calculator: "CombinedPredictionCalculator"
// input_stream: "classification_list_0"
// input_stream: "classification_list_1"
// output_stream: "PREDICTION:prediction"
// options {
// [mediapipe.CombinedPredictionCalculatorOptions.ext] {
// class {
// label: "A"
// score_threshold: 0.7
// }
// default_global_threshold: 0.1
// background_label: "B"
// }
// }
// }
class CombinedPredictionCalculator : public Node {
public:
static constexpr Input<ClassificationList>::Multiple kClassificationListIn{
""};
static constexpr Output<ClassificationList> kPredictionOut{"PREDICTION"};
MEDIAPIPE_NODE_CONTRACT(kClassificationListIn, kPredictionOut);
absl::Status Open(CalculatorContext* cc) override {
options_ = cc->Options<CombinedPredictionCalculatorOptions>();
for (const auto& input : options_.class_()) {
classwise_thresholds_[input.label()] = input.score_threshold();
}
classwise_thresholds_[options_.background_label()] = 0;
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
// After loop, if have winning prediction return. Otherwise empty packet.
std::unique_ptr<ClassificationList> first_winning_prediction = nullptr;
auto collection = kClassificationListIn(cc);
for (const auto& input : collection) {
if (input.IsEmpty() || input.Get().classification_size() == 0) {
continue;
}
auto prediction = GetWinningPrediction(
input.Get(), classwise_thresholds_, options_.background_label(),
options_.default_global_threshold());
if (prediction->classification(0).label() !=
options_.background_label()) {
kPredictionOut(cc).Send(std::move(prediction));
return absl::OkStatus();
}
if (first_winning_prediction == nullptr) {
first_winning_prediction = std::move(prediction);
}
}
if (first_winning_prediction != nullptr) {
kPredictionOut(cc).Send(std::move(first_winning_prediction));
}
return absl::OkStatus();
}
private:
CombinedPredictionCalculatorOptions options_;
absl::btree_map<std::string, float> classwise_thresholds_;
};
MEDIAPIPE_REGISTER_NODE(CombinedPredictionCalculator);
} // namespace api2
} // namespace mediapipe
mediapipe/tasks/cc/vision/gesture_recognizer/calculators/combined_prediction_calculator.proto
+41
View File
@@ -0,0 +1,41 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message CombinedPredictionCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional CombinedPredictionCalculatorOptions ext = 483738635;
}
message Class {
optional string label = 1;
optional float score_threshold = 2;
}
// List of classes with score thresholds.
repeated Class class = 1;
// Default score threshold applied to a label.
optional float default_global_threshold = 2 [default = 0];
// Name of the background class whose input scores will be ignored while
// thresholding.
optional string background_label = 3;
}
mediapipe/tasks/cc/vision/gesture_recognizer/calculators/combined_prediction_calculator_test.cc
+315
View File
@@ -0,0 +1,315 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cmath>
#include <cstdint>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/port/status_matchers.h"
namespace mediapipe {
namespace {
constexpr char kPredictionTag[] = "PREDICTION";
std::unique_ptr<CalculatorRunner> BuildNodeRunnerWithOptions(
float drama_thresh, float llama_thresh, float bazinga_thresh,
float joy_thresh, float peace_thresh) {
constexpr absl::string_view kCalculatorProto = R"pb(
calculator: "CombinedPredictionCalculator"
input_stream: "custom_softmax_scores"
input_stream: "canned_softmax_scores"
output_stream: "PREDICTION:prediction"
options {
[mediapipe.CombinedPredictionCalculatorOptions.ext] {
class { label: "CustomDrama" score_threshold: $0 }
class { label: "CustomLlama" score_threshold: $1 }
class { label: "CannedBazinga" score_threshold: $2 }
class { label: "CannedJoy" score_threshold: $3 }
class { label: "CannedPeace" score_threshold: $4 }
background_label: "Negative"
}
}
)pb";
auto runner = std::make_unique<CalculatorRunner>(
absl::Substitute(kCalculatorProto, drama_thresh, llama_thresh,
bazinga_thresh, joy_thresh, peace_thresh));
return runner;
}
std::unique_ptr<ClassificationList> BuildCustomScoreInput(
const float negative_score, const float drama_score,
const float llama_score) {
auto custom_scores = std::make_unique<ClassificationList>();
auto custom_negative = custom_scores->add_classification();
custom_negative->set_label("Negative");
custom_negative->set_score(negative_score);
auto drama = custom_scores->add_classification();
drama->set_label("CustomDrama");
drama->set_score(drama_score);
auto llama = custom_scores->add_classification();
llama->set_label("CustomLlama");
llama->set_score(llama_score);
return custom_scores;
}
std::unique_ptr<ClassificationList> BuildCannedScoreInput(
const float negative_score, const float bazinga_score,
const float joy_score, const float peace_score) {
auto canned_scores = std::make_unique<ClassificationList>();
auto canned_negative = canned_scores->add_classification();
canned_negative->set_label("Negative");
canned_negative->set_score(negative_score);
auto bazinga = canned_scores->add_classification();
bazinga->set_label("CannedBazinga");
bazinga->set_score(bazinga_score);
auto joy = canned_scores->add_classification();
joy->set_label("CannedJoy");
joy->set_score(joy_score);
auto peace = canned_scores->add_classification();
peace->set_label("CannedPeace");
peace->set_score(peace_score);
return canned_scores;
}
TEST(CombinedPredictionCalculatorPacketTest,
CustomEmpty_CannedEmpty_ResultIsEmpty) {
auto runner = BuildNodeRunnerWithOptions(
/*drama_thresh=*/0.0, /*llama_thresh=*/0.0, /*bazinga_thresh=*/0.0,
/*joy_thresh=*/0.0, /*peace_thresh=*/0.0);
MP_ASSERT_OK(runner->Run()) << "Calculator execution failed.";
EXPECT_THAT(runner->Outputs().Tag("PREDICTION").packets, testing::IsEmpty());
}
TEST(CombinedPredictionCalculatorPacketTest,
CustomEmpty_CannedNotEmpty_ResultIsCanned) {
auto runner = BuildNodeRunnerWithOptions(
/*drama_thresh=*/0.0, /*llama_thresh=*/0.0, /*bazinga_thresh=*/0.9,
/*joy_thresh=*/0.5, /*peace_thresh=*/0.8);
auto canned_scores = BuildCannedScoreInput(
/*negative_score=*/0.1,
/*bazinga_score=*/0.1, /*joy_score=*/0.6, /*peace_score=*/0.2);
runner->MutableInputs()->Index(1).packets.push_back(
Adopt(canned_scores.release()).At(Timestamp(1)));
MP_ASSERT_OK(runner->Run()) << "Calculator execution failed.";
auto output_prediction_packets =
runner->Outputs().Tag(kPredictionTag).packets;
ASSERT_EQ(output_prediction_packets.size(), 1);
Classification output_prediction =
output_prediction_packets[0].Get<ClassificationList>().classification(0);
EXPECT_EQ(output_prediction.label(), "CannedJoy");
EXPECT_NEAR(output_prediction.score(), 0.6, 1e-4);
}
TEST(CombinedPredictionCalculatorPacketTest,
CustomNotEmpty_CannedEmpty_ResultIsCustom) {
auto runner = BuildNodeRunnerWithOptions(
/*drama_thresh=*/0.3, /*llama_thresh=*/0.5, /*bazinga_thresh=*/0.0,
/*joy_thresh=*/0.0, /*peace_thresh=*/0.0);
auto custom_scores =
BuildCustomScoreInput(/*negative_score=*/0.1,
/*drama_score=*/0.2, /*llama_score=*/0.7);
runner->MutableInputs()->Index(0).packets.push_back(
Adopt(custom_scores.release()).At(Timestamp(1)));
MP_ASSERT_OK(runner->Run()) << "Calculator execution failed.";
auto output_prediction_packets =
runner->Outputs().Tag(kPredictionTag).packets;
ASSERT_EQ(output_prediction_packets.size(), 1);
Classification output_prediction =
output_prediction_packets[0].Get<ClassificationList>().classification(0);
EXPECT_EQ(output_prediction.label(), "CustomLlama");
EXPECT_NEAR(output_prediction.score(), 0.7, 1e-4);
}
struct CombinedPredictionCalculatorTestCase {
std::string test_name;
float custom_negative_score;
float drama_score;
float llama_score;
float drama_thresh;
float llama_thresh;
float canned_negative_score;
float bazinga_score;
float joy_score;
float peace_score;
float bazinga_thresh;
float joy_thresh;
float peace_thresh;
std::string max_scoring_label;
float max_score;
};
using CombinedPredictionCalculatorTest =
testing::TestWithParam<CombinedPredictionCalculatorTestCase>;
TEST_P(CombinedPredictionCalculatorTest, OutputsCorrectResult) {
const CombinedPredictionCalculatorTestCase& test_case = GetParam();
auto runner = BuildNodeRunnerWithOptions(
test_case.drama_thresh, test_case.llama_thresh, test_case.bazinga_thresh,
test_case.joy_thresh, test_case.peace_thresh);
auto custom_scores =
BuildCustomScoreInput(test_case.custom_negative_score,
test_case.drama_score, test_case.llama_score);
runner->MutableInputs()->Index(0).packets.push_back(
Adopt(custom_scores.release()).At(Timestamp(1)));
auto canned_scores = BuildCannedScoreInput(
test_case.canned_negative_score, test_case.bazinga_score,
test_case.joy_score, test_case.peace_score);
runner->MutableInputs()->Index(1).packets.push_back(
Adopt(canned_scores.release()).At(Timestamp(1)));
MP_ASSERT_OK(runner->Run()) << "Calculator execution failed.";
auto output_prediction_packets =
runner->Outputs().Tag(kPredictionTag).packets;
ASSERT_EQ(output_prediction_packets.size(), 1);
Classification output_prediction =
output_prediction_packets[0].Get<ClassificationList>().classification(0);
EXPECT_EQ(output_prediction.label(), test_case.max_scoring_label);
EXPECT_NEAR(output_prediction.score(), test_case.max_score, 1e-4);
}
INSTANTIATE_TEST_CASE_P(
CombinedPredictionCalculatorTests, CombinedPredictionCalculatorTest,
testing::ValuesIn<CombinedPredictionCalculatorTestCase>({
{
.test_name = "TestCustomDramaWinnnerWith_HighCanned_Thresh",
.custom_negative_score = 0.1,
.drama_score = 0.5,
.llama_score = 0.3,
.drama_thresh = 0.25,
.llama_thresh = 0.7,
.canned_negative_score = 0.1,
.bazinga_score = 0.3,
.joy_score = 0.3,
.peace_score = 0.3,
.bazinga_thresh = 0.7,
.joy_thresh = 0.7,
.peace_thresh = 0.7,
.max_scoring_label = "CustomDrama",
.max_score = 0.5,
},
{
.test_name = "TestCannedWinnerWith_HighCustom_ZeroCanned_Thresh",
.custom_negative_score = 0.1,
.drama_score = 0.3,
.llama_score = 0.6,
.drama_thresh = 0.4,
.llama_thresh = 0.8,
.canned_negative_score = 0.1,
.bazinga_score = 0.4,
.joy_score = 0.3,
.peace_score = 0.2,
.bazinga_thresh = 0.0,
.joy_thresh = 0.0,
.peace_thresh = 0.0,
.max_scoring_label = "CannedBazinga",
.max_score = 0.4,
},
{
.test_name = "TestNegativeWinnerWith_LowCustom_HighCanned_Thresh",
.custom_negative_score = 0.5,
.drama_score = 0.1,
.llama_score = 0.4,
.drama_thresh = 0.1,
.llama_thresh = 0.05,
.canned_negative_score = 0.1,
.bazinga_score = 0.3,
.joy_score = 0.3,
.peace_score = 0.3,
.bazinga_thresh = 0.7,
.joy_thresh = 0.7,
.peace_thresh = 0.7,
.max_scoring_label = "Negative",
.max_score = 0.5,
},
{
.test_name = "TestNegativeWinnerWith_HighCustom_HighCanned_Thresh",
.custom_negative_score = 0.8,
.drama_score = 0.1,
.llama_score = 0.1,
.drama_thresh = 0.25,
.llama_thresh = 0.7,
.canned_negative_score = 0.1,
.bazinga_score = 0.3,
.joy_score = 0.3,
.peace_score = 0.3,
.bazinga_thresh = 0.7,
.joy_thresh = 0.7,
.peace_thresh = 0.7,
.max_scoring_label = "Negative",
.max_score = 0.8,
},
{
.test_name = "TestNegativeWinnerWith_HighCustom_HighCannedThresh2",
.custom_negative_score = 0.1,
.drama_score = 0.2,
.llama_score = 0.7,
.drama_thresh = 1.1,
.llama_thresh = 1.1,
.canned_negative_score = 0.1,
.bazinga_score = 0.3,
.joy_score = 0.3,
.peace_score = 0.3,
.bazinga_thresh = 0.7,
.joy_thresh = 0.7,
.peace_thresh = 0.7,
.max_scoring_label = "Negative",
.max_score = 0.1,
},
{
.test_name = "TestNegativeWinnerWith_HighCustom_HighCanned_Thresh3",
.custom_negative_score = 0.1,
.drama_score = 0.3,
.llama_score = 0.6,
.drama_thresh = 0.4,
.llama_thresh = 0.8,
.canned_negative_score = 0.3,
.bazinga_score = 0.2,
.joy_score = 0.3,
.peace_score = 0.2,
.bazinga_thresh = 0.5,
.joy_thresh = 0.5,
.peace_thresh = 0.5,
.max_scoring_label = "Negative",
.max_score = 0.1,
},
}),
[](const testing::TestParamInfo<
CombinedPredictionCalculatorTest::ParamType>& info) {
return info.param.test_name;
});
} // namespace
} // namespace mediapipe
mediapipe/tasks/cc/vision/gesture_recognizer/calculators/landmarks_to_matrix_calculator.cc
+32
View File
@@ -14,6 +14,7 @@ limitations under the License.
==============================================================================*/
#include <algorithm>
#include <cmath>
#include <limits>
#include <memory>
#include <string>
@@ -26,6 +27,7 @@ limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/calculators/landmarks_to_matrix_calculator.pb.h"
@@ -38,6 +40,7 @@ namespace {
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kWorldLandmarksTag[] = "WORLD_LANDMARKS";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kLandmarksMatrixTag[] = "LANDMARKS_MATRIX";
constexpr int kFeaturesPerLandmark = 3;
@@ -62,6 +65,25 @@ absl::StatusOr<LandmarkListT> NormalizeLandmarkAspectRatio(
return normalized_landmarks;
}
template <class LandmarkListT>
absl::StatusOr<LandmarkListT> RotateLandmarks(const LandmarkListT& landmarks,
float rotation) {
float cos = std::cos(rotation);
// Negate because Y-axis points down and not up.
float sin = std::sin(-rotation);
LandmarkListT rotated_landmarks;
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const auto& old_landmark = landmarks.landmark(i);
float x = old_landmark.x() - 0.5;
float y = old_landmark.y() - 0.5;
auto* new_landmark = rotated_landmarks.add_landmark();
new_landmark->set_x(x * cos - y * sin + 0.5);
new_landmark->set_y(y * cos + x * sin + 0.5);
new_landmark->set_z(old_landmark.z());
}
return rotated_landmarks;
}
template <class LandmarkListT>
absl::StatusOr<LandmarkListT> NormalizeObject(const LandmarkListT& landmarks,
int origin_offset) {
@@ -134,6 +156,13 @@ absl::Status ProcessLandmarks(LandmarkListT landmarks, CalculatorContext* cc) {
NormalizeLandmarkAspectRatio(landmarks, width, height));
}
if (cc->Inputs().HasTag(kNormRectTag)) {
RET_CHECK(!cc->Inputs().Tag(kNormRectTag).IsEmpty());
const auto rotation =
cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>().rotation();
ASSIGN_OR_RETURN(landmarks, RotateLandmarks(landmarks, rotation));
}
const auto& options = cc->Options<LandmarksToMatrixCalculatorOptions>();
if (options.object_normalization()) {
ASSIGN_OR_RETURN(
@@ -163,6 +192,8 @@ absl::Status ProcessLandmarks(LandmarkListT landmarks, CalculatorContext* cc) {
// WORLD_LANDMARKS - World 3d landmarks of one object. Use *either*
// LANDMARKS or WORLD_LANDMARKS.
// IMAGE_SIZE - (width, height) of the image
// NORM_RECT - Optional NormalizedRect object whose 'rotation' field is used
// to rotate the landmarks.
// Output:
// LANDMARKS_MATRIX - Matrix for the landmarks.
//
@@ -185,6 +216,7 @@ class LandmarksToMatrixCalculator : public CalculatorBase {
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>().Optional();
cc->Inputs().Tag(kWorldLandmarksTag).Set<LandmarkList>().Optional();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>().Optional();
cc->Inputs().Tag(kNormRectTag).Set<NormalizedRect>().Optional();
cc->Outputs().Tag(kLandmarksMatrixTag).Set<Matrix>();
return absl::OkStatus();
}
mediapipe/tasks/cc/vision/gesture_recognizer/calculators/landmarks_to_matrix_calculator_test.cc
+39 -5
View File
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cmath>
#include <memory>
#include <string>
#include <utility>
@@ -23,6 +24,7 @@ limitations under the License.
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
@@ -35,6 +37,7 @@ constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kWorldLandmarksTag[] = "WORLD_LANDMARKS";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kLandmarksMatrixTag[] = "LANDMARKS_MATRIX";
constexpr char kNormRectTag[] = "NORM_RECT";
template <class LandmarkListT>
LandmarkListT BuildPseudoLandmarks(int num_landmarks, int offset = 0) {
@@ -54,6 +57,7 @@ struct Landmarks2dToMatrixCalculatorTestCase {
int object_normalization_origin_offset = -1;
float expected_cell_0_2;
float expected_cell_1_5;
float rotation;
};
using Landmarks2dToMatrixCalculatorTest =
@@ -68,6 +72,7 @@ TEST_P(Landmarks2dToMatrixCalculatorTest, OutputsCorrectResult) {
calculator: "LandmarksToMatrixCalculator"
input_stream: "LANDMARKS:landmarks"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "NORM_RECT:norm_rect"
output_stream: "LANDMARKS_MATRIX:landmarks_matrix"
options {
[mediapipe.LandmarksToMatrixCalculatorOptions.ext] {
@@ -91,6 +96,11 @@ TEST_P(Landmarks2dToMatrixCalculatorTest, OutputsCorrectResult) {
runner.MutableInputs()
->Tag(kImageSizeTag)
.packets.push_back(Adopt(image_size.release()).At(Timestamp(0)));
auto norm_rect = std::make_unique<NormalizedRect>();
norm_rect->set_rotation(test_case.rotation);
runner.MutableInputs()
->Tag(kNormRectTag)
.packets.push_back(Adopt(norm_rect.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
@@ -109,12 +119,20 @@ INSTANTIATE_TEST_CASE_P(
.base_offset = 0,
.object_normalization_origin_offset = 0,
.expected_cell_0_2 = 0.1f,
.expected_cell_1_5 = 0.1875f},
.expected_cell_1_5 = 0.1875f,
.rotation = 0},
{.test_name = "TestWithOffset21",
.base_offset = 21,
.object_normalization_origin_offset = 0,
.expected_cell_0_2 = 0.1f,
.expected_cell_1_5 = 0.1875f}}),
.expected_cell_1_5 = 0.1875f,
.rotation = 0},
{.test_name = "TestWithRotation",
.base_offset = 0,
.object_normalization_origin_offset = 0,
.expected_cell_0_2 = 0.075f,
.expected_cell_1_5 = -0.25f,
.rotation = M_PI / 2.0}}),
[](const testing::TestParamInfo<
Landmarks2dToMatrixCalculatorTest::ParamType>& info) {
return info.param.test_name;
@@ -126,6 +144,7 @@ struct LandmarksWorld3dToMatrixCalculatorTestCase {
int object_normalization_origin_offset = -1;
float expected_cell_0_2;
float expected_cell_1_5;
float rotation;
};
using LandmarksWorld3dToMatrixCalculatorTest =
@@ -140,6 +159,7 @@ TEST_P(LandmarksWorld3dToMatrixCalculatorTest, OutputsCorrectResult) {
calculator: "LandmarksToMatrixCalculator"
input_stream: "WORLD_LANDMARKS:landmarks"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "NORM_RECT:norm_rect"
output_stream: "LANDMARKS_MATRIX:landmarks_matrix"
options {
[mediapipe.LandmarksToMatrixCalculatorOptions.ext] {
@@ -162,6 +182,11 @@ TEST_P(LandmarksWorld3dToMatrixCalculatorTest, OutputsCorrectResult) {
runner.MutableInputs()
->Tag(kImageSizeTag)
.packets.push_back(Adopt(image_size.release()).At(Timestamp(0)));
auto norm_rect = std::make_unique<NormalizedRect>();
norm_rect->set_rotation(test_case.rotation);
runner.MutableInputs()
->Tag(kNormRectTag)
.packets.push_back(Adopt(norm_rect.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
@@ -180,17 +205,26 @@ INSTANTIATE_TEST_CASE_P(
.base_offset = 0,
.object_normalization_origin_offset = 0,
.expected_cell_0_2 = 0.1f,
.expected_cell_1_5 = 0.25},
.expected_cell_1_5 = 0.25,
.rotation = 0},
{.test_name = "TestWithOffset21",
.base_offset = 21,
.object_normalization_origin_offset = 0,
.expected_cell_0_2 = 0.1f,
.expected_cell_1_5 = 0.25},
.expected_cell_1_5 = 0.25,
.rotation = 0},
{.test_name = "NoObjectNormalization",
.base_offset = 0,
.object_normalization_origin_offset = -1,
.expected_cell_0_2 = 0.021f,
.expected_cell_1_5 = 0.052f}}),
.expected_cell_1_5 = 0.052f,
.rotation = 0},
{.test_name = "TestWithRotation",
.base_offset = 0,
.object_normalization_origin_offset = 0,
.expected_cell_0_2 = 0.1f,
.expected_cell_1_5 = -0.25f,
.rotation = M_PI / 2.0}}),
[](const testing::TestParamInfo<
LandmarksWorld3dToMatrixCalculatorTest::ParamType>& info) {
return info.param.test_name;
mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.cc
+64 -44
View File
@@ -17,6 +17,7 @@ limitations under the License.
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
@@ -27,6 +28,7 @@ limitations under the License.
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/tasks/cc/common.h"
#include "mediapipe/tasks/cc/components/image_preprocessing.h"
@@ -37,7 +39,9 @@ limitations under the License.
#include "mediapipe/tasks/cc/core/task_runner.h"
#include "mediapipe/tasks/cc/core/utils.h"
#include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/vision_task_api_factory.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_classifier_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/hand_detector/proto/hand_detector_graph_options.pb.h"
@@ -62,6 +66,8 @@ constexpr char kHandGestureSubgraphTypeName[] =
constexpr char kImageTag[] = "IMAGE";
constexpr char kImageInStreamName[] = "image_in";
constexpr char kImageOutStreamName[] = "image_out";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kNormRectStreamName[] = "norm_rect_in";
constexpr char kHandGesturesTag[] = "HAND_GESTURES";
constexpr char kHandGesturesStreamName[] = "hand_gestures";
constexpr char kHandednessTag[] = "HANDEDNESS";
@@ -83,6 +89,7 @@ CalculatorGraphConfig CreateGraphConfig(
auto& subgraph = graph.AddNode(kHandGestureSubgraphTypeName);
subgraph.GetOptions<GestureRecognizerGraphOptionsProto>().Swap(options.get());
graph.In(kImageTag).SetName(kImageInStreamName);
graph.In(kNormRectTag).SetName(kNormRectStreamName);
subgraph.Out(kHandGesturesTag).SetName(kHandGesturesStreamName) >>
graph.Out(kHandGesturesTag);
subgraph.Out(kHandednessTag).SetName(kHandednessStreamName) >>
@@ -93,10 +100,11 @@ CalculatorGraphConfig CreateGraphConfig(
graph.Out(kHandWorldLandmarksTag);
subgraph.Out(kImageTag).SetName(kImageOutStreamName) >> graph.Out(kImageTag);
if (enable_flow_limiting) {
return tasks::core::AddFlowLimiterCalculator(graph, subgraph, {kImageTag},
kHandGesturesTag);
return tasks::core::AddFlowLimiterCalculator(
graph, subgraph, {kImageTag, kNormRectTag}, kHandGesturesTag);
}
graph.In(kImageTag) >> subgraph.In(kImageTag);
graph.In(kNormRectTag) >> subgraph.In(kNormRectTag);
return graph.GetConfig();
}
@@ -105,59 +113,50 @@ CalculatorGraphConfig CreateGraphConfig(
std::unique_ptr<GestureRecognizerGraphOptionsProto>
ConvertGestureRecognizerGraphOptionsProto(GestureRecognizerOptions* options) {
auto options_proto = std::make_unique<GestureRecognizerGraphOptionsProto>();
auto base_options_proto = std::make_unique<tasks::core::proto::BaseOptions>(
tasks::core::ConvertBaseOptionsToProto(&(options->base_options)));
options_proto->mutable_base_options()->Swap(base_options_proto.get());
options_proto->mutable_base_options()->set_use_stream_mode(
options->running_mode != core::RunningMode::IMAGE);
bool use_stream_mode = options->running_mode != core::RunningMode::IMAGE;
// TODO remove these workarounds for base options of subgraphs.
// Configure hand detector options.
auto base_options_proto_for_hand_detector =
std::make_unique<tasks::core::proto::BaseOptions>(
tasks::core::ConvertBaseOptionsToProto(
&(options->base_options_for_hand_detector)));
base_options_proto_for_hand_detector->set_use_stream_mode(use_stream_mode);
auto* hand_detector_graph_options =
options_proto->mutable_hand_landmarker_graph_options()
->mutable_hand_detector_graph_options();
hand_detector_graph_options->mutable_base_options()->Swap(
base_options_proto_for_hand_detector.get());
hand_detector_graph_options->set_num_hands(options->num_hands);
hand_detector_graph_options->set_min_detection_confidence(
options->min_hand_detection_confidence);
// Configure hand landmark detector options.
auto base_options_proto_for_hand_landmarker =
std::make_unique<tasks::core::proto::BaseOptions>(
tasks::core::ConvertBaseOptionsToProto(
&(options->base_options_for_hand_landmarker)));
base_options_proto_for_hand_landmarker->set_use_stream_mode(use_stream_mode);
auto* hand_landmarks_detector_graph_options =
options_proto->mutable_hand_landmarker_graph_options()
->mutable_hand_landmarks_detector_graph_options();
hand_landmarks_detector_graph_options->mutable_base_options()->Swap(
base_options_proto_for_hand_landmarker.get());
hand_landmarks_detector_graph_options->set_min_detection_confidence(
options->min_hand_presence_confidence);
auto* hand_landmarker_graph_options =
options_proto->mutable_hand_landmarker_graph_options();
hand_landmarker_graph_options->set_min_tracking_confidence(
options->min_tracking_confidence);
auto* hand_landmarks_detector_graph_options =
hand_landmarker_graph_options
->mutable_hand_landmarks_detector_graph_options();
hand_landmarks_detector_graph_options->set_min_detection_confidence(
options->min_hand_presence_confidence);
// Configure hand gesture recognizer options.
auto base_options_proto_for_gesture_recognizer =
std::make_unique<tasks::core::proto::BaseOptions>(
tasks::core::ConvertBaseOptionsToProto(
&(options->base_options_for_gesture_recognizer)));
base_options_proto_for_gesture_recognizer->set_use_stream_mode(
use_stream_mode);
auto* hand_gesture_recognizer_graph_options =
options_proto->mutable_hand_gesture_recognizer_graph_options();
hand_gesture_recognizer_graph_options->mutable_base_options()->Swap(
base_options_proto_for_gesture_recognizer.get());
if (options->min_gesture_confidence >= 0) {
hand_gesture_recognizer_graph_options->mutable_classifier_options()
->set_score_threshold(options->min_gesture_confidence);
}
auto canned_gestures_classifier_options_proto =
std::make_unique<components::processors::proto::ClassifierOptions>(
components::processors::ConvertClassifierOptionsToProto(
&(options->canned_gestures_classifier_options)));
hand_gesture_recognizer_graph_options
->mutable_canned_gesture_classifier_graph_options()
->mutable_classifier_options()
->Swap(canned_gestures_classifier_options_proto.get());
auto custom_gestures_classifier_options_proto =
std::make_unique<components::processors::proto::ClassifierOptions>(
components::processors::ConvertClassifierOptionsToProto(
&(options->canned_gestures_classifier_options)));
hand_gesture_recognizer_graph_options
->mutable_custom_gesture_classifier_graph_options()
->mutable_classifier_options()
->Swap(canned_gestures_classifier_options_proto.get());
return options_proto;
}
@@ -216,16 +215,23 @@ absl::StatusOr<std::unique_ptr<GestureRecognizer>> GestureRecognizer::Create(
}
absl::StatusOr<GestureRecognitionResult> GestureRecognizer::Recognize(
mediapipe::Image image) {
mediapipe::Image image,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"GPU input images are currently not supported.",
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(auto output_packets,
ProcessImageData({{kImageInStreamName,
MakePacket<Image>(std::move(image))}}));
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
{{kImageInStreamName, MakePacket<Image>(std::move(image))},
{kNormRectStreamName,
MakePacket<NormalizedRect>(std::move(norm_rect))}}));
if (output_packets[kHandGesturesStreamName].IsEmpty()) {
return {{{}, {}, {}, {}}};
}
@@ -245,18 +251,25 @@ absl::StatusOr<GestureRecognitionResult> GestureRecognizer::Recognize(
}
absl::StatusOr<GestureRecognitionResult> GestureRecognizer::RecognizeForVideo(
mediapipe::Image image, int64 timestamp_ms) {
mediapipe::Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))
.At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))},
{kNormRectStreamName,
MakePacket<NormalizedRect>(std::move(norm_rect))
.At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}}));
if (output_packets[kHandGesturesStreamName].IsEmpty()) {
return {{{}, {}, {}, {}}};
@@ -276,17 +289,24 @@ absl::StatusOr<GestureRecognitionResult> GestureRecognizer::RecognizeForVideo(
};
}
absl::Status GestureRecognizer::RecognizeAsync(mediapipe::Image image,
int64 timestamp_ms) {
absl::Status GestureRecognizer::RecognizeAsync(
mediapipe::Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))
.At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))},
{kNormRectStreamName,
MakePacket<NormalizedRect>(std::move(norm_rect))
.At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}});
}
mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.h
+50 -19
View File
@@ -17,14 +17,18 @@ limitations under the License.
#define MEDIAPIPE_TASKS_CC_VISION_GESTURE_RECOGNIZRER_GESTURE_RECOGNIZER_H_
#include <memory>
#include <optional>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/tasks/cc/components/containers/gesture_recognition_result.h"
#include "mediapipe/tasks/cc/components/processors/classifier_options.h"
#include "mediapipe/tasks/cc/core/base_options.h"
#include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
namespace mediapipe {
@@ -37,12 +41,6 @@ struct GestureRecognizerOptions {
// model file with metadata, accelerator options, op resolver, etc.
tasks::core::BaseOptions base_options;
// TODO: remove these. Temporary solutions before bundle asset is
// ready.
tasks::core::BaseOptions base_options_for_hand_landmarker;
tasks::core::BaseOptions base_options_for_hand_detector;
tasks::core::BaseOptions base_options_for_gesture_recognizer;
// The running mode of the task. Default to the image mode.
// GestureRecognizer has three running modes:
// 1) The image mode for recognizing hand gestures on single image inputs.
@@ -57,7 +55,7 @@ struct GestureRecognizerOptions {
int num_hands = 1;
// The minimum confidence score for the hand detection to be considered
// successfully.
// successful.
float min_hand_detection_confidence = 0.5;
// The minimum confidence score of hand presence score in the hand landmark
@@ -65,15 +63,20 @@ struct GestureRecognizerOptions {
float min_hand_presence_confidence = 0.5;
// The minimum confidence score for the hand tracking to be considered
// successfully.
// successful.
float min_tracking_confidence = 0.5;
// The minimum confidence score for the gestures to be considered
// successfully. If < 0, the gesture confidence thresholds in the model
// metadata are used.
// TODO Note this option is subject to change, after scoring
// merging calculator is implemented.
float min_gesture_confidence = -1;
// TODO Note this option is subject to change.
// Options for configuring the canned gestures classifier, such as score
// threshold, allow list and deny list of gestures. The categories for canned
// gesture classifiers are: ["None", "Closed_Fist", "Open_Palm",
// "Pointing_Up", "Thumb_Down", "Thumb_Up", "Victory", "ILoveYou"]
components::processors::ClassifierOptions canned_gestures_classifier_options;
// TODO Note this option is subject to change.
// Options for configuring the custom gestures classifier, such as score
// threshold, allow list and deny list of gestures.
components::processors::ClassifierOptions custom_gestures_classifier_options;
// The user-defined result callback for processing live stream data.
// The result callback should only be specified when the running mode is set
@@ -93,6 +96,13 @@ struct GestureRecognizerOptions {
// Inputs:
// Image
// - The image that gesture recognition runs on.
// std::optional<NormalizedRect>
// - If provided, can be used to specify the rotation to apply to the image
// before performing gesture recognition, by setting its 'rotation' field
// in radians (e.g. 'M_PI / 2' for a 90° anti-clockwise rotation). Note
// that specifying a region-of-interest using the 'x_center', 'y_center',
// 'width' and 'height' fields is NOT supported and will result in an
// invalid argument error being returned.
// Outputs:
// GestureRecognitionResult
// - The hand gesture recognition results.
@@ -120,24 +130,37 @@ class GestureRecognizer : tasks::vision::core::BaseVisionTaskApi {
// Only use this method when the GestureRecognizer is created with the image
// running mode.
//
// image - mediapipe::Image
// Image to perform hand gesture recognition on.
// The optional 'image_processing_options' parameter can be used to specify
// the rotation to apply to the image before performing recognition, by
// setting its 'rotation_degrees' field. Note that specifying a
// region-of-interest using the 'region_of_interest' field is NOT supported
// and will result in an invalid argument error being returned.
//
// The image can be of any size with format RGB or RGBA.
// TODO: Describes how the input image will be preprocessed
// after the yuv support is implemented.
absl::StatusOr<components::containers::GestureRecognitionResult> Recognize(
Image image);
Image image,
std::optional<core::ImageProcessingOptions> image_processing_options =
std::nullopt);
// Performs gesture recognition on the provided video frame.
// Only use this method when the GestureRecognizer is created with the video
// running mode.
//
// The optional 'image_processing_options' parameter can be used to specify
// the rotation to apply to the image before performing recognition, by
// setting its 'rotation_degrees' field. Note that specifying a
// region-of-interest using the 'region_of_interest' field is NOT supported
// and will result in an invalid argument error being returned.
//
// The image can be of any size with format RGB or RGBA. It's required to
// provide the video frame's timestamp (in milliseconds). The input timestamps
// must be monotonically increasing.
absl::StatusOr<components::containers::GestureRecognitionResult>
RecognizeForVideo(Image image, int64 timestamp_ms);
RecognizeForVideo(Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions>
image_processing_options = std::nullopt);
// Sends live image data to perform gesture recognition, and the results will
// be available via the "result_callback" provided in the
@@ -149,6 +172,12 @@ class GestureRecognizer : tasks::vision::core::BaseVisionTaskApi {
// sent to the gesture recognizer. The input timestamps must be monotonically
// increasing.
//
// The optional 'image_processing_options' parameter can be used to specify
// the rotation to apply to the image before performing recognition, by
// setting its 'rotation_degrees' field. Note that specifying a
// region-of-interest using the 'region_of_interest' field is NOT supported
// and will result in an invalid argument error being returned.
//
// The "result_callback" provides
// - A vector of GestureRecognitionResult, each is the recognized results
// for a input frame.
@@ -157,7 +186,9 @@ class GestureRecognizer : tasks::vision::core::BaseVisionTaskApi {
// longer be valid when the callback returns. To access the image data
// outside of the callback, callers need to make a copy of the image.
// - The input timestamp in milliseconds.
absl::Status RecognizeAsync(Image image, int64 timestamp_ms);
absl::Status RecognizeAsync(Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions>
image_processing_options = std::nullopt);
// Shuts down the GestureRecognizer when all works are done.
absl::Status Close() { return runner_->Close(); }
mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc
+84 -3
View File
@@ -24,9 +24,14 @@ limitations under the License.
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/tasks/cc/common.h"
#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h"
#include "mediapipe/tasks/cc/core/model_resources_cache.h"
#include "mediapipe/tasks/cc/core/model_task_graph.h"
#include "mediapipe/tasks/cc/core/utils.h"
#include "mediapipe/tasks/cc/metadata/utils/zip_utils.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/hand_detector/proto/hand_detector_graph_options.pb.h"
@@ -45,6 +50,8 @@ using ::mediapipe::api2::Input;
using ::mediapipe::api2::Output;
using ::mediapipe::api2::builder::Graph;
using ::mediapipe::api2::builder::Source;
using ::mediapipe::tasks::core::ModelAssetBundleResources;
using ::mediapipe::tasks::metadata::SetExternalFile;
using ::mediapipe::tasks::vision::gesture_recognizer::proto::
GestureRecognizerGraphOptions;
using ::mediapipe::tasks::vision::gesture_recognizer::proto::
@@ -53,12 +60,16 @@ using ::mediapipe::tasks::vision::hand_landmarker::proto::
HandLandmarkerGraphOptions;
constexpr char kImageTag[] = "IMAGE";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kWorldLandmarksTag[] = "WORLD_LANDMARKS";
constexpr char kHandednessTag[] = "HANDEDNESS";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kHandGesturesTag[] = "HAND_GESTURES";
constexpr char kHandTrackingIdsTag[] = "HAND_TRACKING_IDS";
constexpr char kHandLandmarkerBundleAssetName[] = "hand_landmarker.task";
constexpr char kHandGestureRecognizerBundleAssetName[] =
"hand_gesture_recognizer.task";
struct GestureRecognizerOutputs {
Source<std::vector<ClassificationList>> gesture;
@@ -68,6 +79,53 @@ struct GestureRecognizerOutputs {
Source<Image> image;
};
// Sets the base options in the sub tasks.
absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
GestureRecognizerGraphOptions* options,
bool is_copy) {
ASSIGN_OR_RETURN(const auto hand_landmarker_file,
resources.GetModelFile(kHandLandmarkerBundleAssetName));
auto* hand_landmarker_graph_options =
options->mutable_hand_landmarker_graph_options();
SetExternalFile(hand_landmarker_file,
hand_landmarker_graph_options->mutable_base_options()
->mutable_model_asset(),
is_copy);
hand_landmarker_graph_options->mutable_base_options()
->mutable_acceleration()
->CopyFrom(options->base_options().acceleration());
hand_landmarker_graph_options->mutable_base_options()->set_use_stream_mode(
options->base_options().use_stream_mode());
ASSIGN_OR_RETURN(
const auto hand_gesture_recognizer_file,
resources.GetModelFile(kHandGestureRecognizerBundleAssetName));
auto* hand_gesture_recognizer_graph_options =
options->mutable_hand_gesture_recognizer_graph_options();
SetExternalFile(hand_gesture_recognizer_file,
hand_gesture_recognizer_graph_options->mutable_base_options()
->mutable_model_asset(),
is_copy);
hand_gesture_recognizer_graph_options->mutable_base_options()
->mutable_acceleration()
->CopyFrom(options->base_options().acceleration());
if (!hand_gesture_recognizer_graph_options->base_options()
.acceleration()
.has_xnnpack() &&
!hand_gesture_recognizer_graph_options->base_options()
.acceleration()
.has_tflite()) {
hand_gesture_recognizer_graph_options->mutable_base_options()
->mutable_acceleration()
->mutable_xnnpack();
LOG(WARNING) << "Hand Gesture Recognizer contains CPU only ops. Sets "
<< "HandGestureRecognizerGraph acceleartion to Xnnpack.";
}
hand_gesture_recognizer_graph_options->mutable_base_options()
->set_use_stream_mode(options->base_options().use_stream_mode());
return absl::OkStatus();
}
} // namespace
// A "mediapipe.tasks.vision.gesture_recognizer.GestureRecognizerGraph" performs
@@ -76,6 +134,9 @@ struct GestureRecognizerOutputs {
// Inputs:
// IMAGE - Image
// Image to perform hand gesture recognition on.
// NORM_RECT - NormalizedRect
// Describes image rotation and region of image to perform landmarks
// detection on.
//
// Outputs:
// HAND_GESTURES - std::vector<ClassificationList>
@@ -93,13 +154,15 @@ struct GestureRecognizerOutputs {
// IMAGE - mediapipe::Image
// The image that gesture recognizer runs on and has the pixel data stored
// on the target storage (CPU vs GPU).
//
// All returned coordinates are in the unrotated and uncropped input image
// coordinates system.
//
// Example:
// node {
// calculator:
// "mediapipe.tasks.vision.gesture_recognizer.GestureRecognizerGraph"
// input_stream: "IMAGE:image_in"
// input_stream: "NORM_RECT:norm_rect"
// output_stream: "HAND_GESTURES:hand_gestures"
// output_stream: "LANDMARKS:hand_landmarks"
// output_stream: "WORLD_LANDMARKS:world_hand_landmarks"
@@ -129,10 +192,26 @@ class GestureRecognizerGraph : public core::ModelTaskGraph {
absl::StatusOr<CalculatorGraphConfig> GetConfig(
SubgraphContext* sc) override {
Graph graph;
if (sc->Options<GestureRecognizerGraphOptions>()
.base_options()
.has_model_asset()) {
ASSIGN_OR_RETURN(
const auto* model_asset_bundle_resources,
CreateModelAssetBundleResources<GestureRecognizerGraphOptions>(sc));
// When the model resources cache service is available, filling in
// the file pointer meta in the subtasks' base options. Otherwise,
// providing the file contents instead.
MP_RETURN_IF_ERROR(SetSubTaskBaseOptions(
*model_asset_bundle_resources,
sc->MutableOptions<GestureRecognizerGraphOptions>(),
!sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService)
.IsAvailable()));
}
ASSIGN_OR_RETURN(auto hand_gesture_recognition_output,
BuildGestureRecognizerGraph(
*sc->MutableOptions<GestureRecognizerGraphOptions>(),
graph[Input<Image>(kImageTag)], graph));
graph[Input<Image>(kImageTag)],
graph[Input<NormalizedRect>(kNormRectTag)], graph));
hand_gesture_recognition_output.gesture >>
graph[Output<std::vector<ClassificationList>>(kHandGesturesTag)];
hand_gesture_recognition_output.handedness >>
@@ -148,7 +227,7 @@ class GestureRecognizerGraph : public core::ModelTaskGraph {
private:
absl::StatusOr<GestureRecognizerOutputs> BuildGestureRecognizerGraph(
GestureRecognizerGraphOptions& graph_options, Source<Image> image_in,
Graph& graph) {
Source<NormalizedRect> norm_rect_in, Graph& graph) {
auto& image_property = graph.AddNode("ImagePropertiesCalculator");
image_in >> image_property.In("IMAGE");
auto image_size = image_property.Out("SIZE");
@@ -162,6 +241,7 @@ class GestureRecognizerGraph : public core::ModelTaskGraph {
graph_options.mutable_hand_landmarker_graph_options());
image_in >> hand_landmarker_graph.In(kImageTag);
norm_rect_in >> hand_landmarker_graph.In(kNormRectTag);
auto hand_landmarks =
hand_landmarker_graph[Output<std::vector<NormalizedLandmarkList>>(
kLandmarksTag)];
@@ -187,6 +267,7 @@ class GestureRecognizerGraph : public core::ModelTaskGraph {
hand_world_landmarks >> hand_gesture_subgraph.In(kWorldLandmarksTag);
handedness >> hand_gesture_subgraph.In(kHandednessTag);
image_size >> hand_gesture_subgraph.In(kImageSizeTag);
norm_rect_in >> hand_gesture_subgraph.In(kNormRectTag);
hand_landmarks_id >> hand_gesture_subgraph.In(kHandTrackingIdsTag);
auto hand_gestures =
hand_gesture_subgraph[Output<std::vector<ClassificationList>>(
mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc
+244 -33
View File
@@ -25,15 +25,25 @@ limitations under the License.
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/tasks/cc/common.h"
#include "mediapipe/tasks/cc/components/processors/classification_postprocessing_graph.h"
#include "mediapipe/tasks/cc/components/processors/proto/classification_postprocessing_graph_options.pb.h"
#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h"
#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h"
#include "mediapipe/tasks/cc/core/model_resources.h"
#include "mediapipe/tasks/cc/core/model_resources_cache.h"
#include "mediapipe/tasks/cc/core/model_task_graph.h"
#include "mediapipe/tasks/cc/core/proto/base_options.pb.h"
#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
#include "mediapipe/tasks/cc/core/proto/inference_subgraph.pb.h"
#include "mediapipe/tasks/cc/core/utils.h"
#include "mediapipe/tasks/cc/metadata/utils/zip_utils.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/calculators/combined_prediction_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/calculators/landmarks_to_matrix_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_classifier_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_embedder_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.pb.h"
#include "mediapipe/tasks/metadata/metadata_schema_generated.h"
@@ -50,6 +60,9 @@ using ::mediapipe::api2::builder::Graph;
using ::mediapipe::api2::builder::Source;
using ::mediapipe::tasks::components::processors::
ConfigureTensorsToClassificationCalculator;
using ::mediapipe::tasks::core::ModelAssetBundleResources;
using ::mediapipe::tasks::core::proto::BaseOptions;
using ::mediapipe::tasks::metadata::SetExternalFile;
using ::mediapipe::tasks::vision::gesture_recognizer::proto::
HandGestureRecognizerGraphOptions;
@@ -57,6 +70,7 @@ constexpr char kHandednessTag[] = "HANDEDNESS";
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kWorldLandmarksTag[] = "WORLD_LANDMARKS";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kHandTrackingIdsTag[] = "HAND_TRACKING_IDS";
constexpr char kHandGesturesTag[] = "HAND_GESTURES";
constexpr char kLandmarksMatrixTag[] = "LANDMARKS_MATRIX";
@@ -68,6 +82,21 @@ constexpr char kVectorTag[] = "VECTOR";
constexpr char kIndexTag[] = "INDEX";
constexpr char kIterableTag[] = "ITERABLE";
constexpr char kBatchEndTag[] = "BATCH_END";
constexpr char kPredictionTag[] = "PREDICTION";
constexpr char kBackgroundLabel[] = "None";
constexpr char kGestureEmbedderTFLiteName[] = "gesture_embedder.tflite";
constexpr char kCannedGestureClassifierTFLiteName[] =
"canned_gesture_classifier.tflite";
constexpr char kCustomGestureClassifierTFLiteName[] =
"custom_gesture_classifier.tflite";
struct SubTaskModelResources {
const core::ModelResources* gesture_embedder_model_resource = nullptr;
const core::ModelResources* canned_gesture_classifier_model_resource =
nullptr;
const core::ModelResources* custom_gesture_classifier_model_resource =
nullptr;
};
Source<std::vector<Tensor>> ConvertMatrixToTensor(Source<Matrix> matrix,
Graph& graph) {
@@ -76,6 +105,21 @@ Source<std::vector<Tensor>> ConvertMatrixToTensor(Source<Matrix> matrix,
return node[Output<std::vector<Tensor>>{"TENSORS"}];
}
absl::Status ConfigureCombinedPredictionCalculator(
CombinedPredictionCalculatorOptions* options) {
options->set_background_label(kBackgroundLabel);
return absl::OkStatus();
}
void PopulateAccelerationAndUseStreamMode(
const BaseOptions& parent_base_options,
BaseOptions* sub_task_base_options) {
sub_task_base_options->mutable_acceleration()->CopyFrom(
parent_base_options.acceleration());
sub_task_base_options->set_use_stream_mode(
parent_base_options.use_stream_mode());
}
} // namespace
// A
@@ -92,6 +136,9 @@ Source<std::vector<Tensor>> ConvertMatrixToTensor(Source<Matrix> matrix,
// Detected hand landmarks in world coordinates.
// IMAGE_SIZE - std::pair<int, int>
// The size of image from which the landmarks detected from.
// NORM_RECT - NormalizedRect
// NormalizedRect whose 'rotation' field is used to rotate the
// landmarks before processing them.
//
// Outputs:
// HAND_GESTURES - ClassificationList
@@ -106,6 +153,7 @@ Source<std::vector<Tensor>> ConvertMatrixToTensor(Source<Matrix> matrix,
// input_stream: "LANDMARKS:landmarks"
// input_stream: "WORLD_LANDMARKS:world_landmarks"
// input_stream: "IMAGE_SIZE:image_size"
// input_stream: "NORM_RECT:norm_rect"
// output_stream: "HAND_GESTURES:hand_gestures"
// options {
// [mediapipe.tasks.vision.gesture_recognizer.proto.HandGestureRecognizerGraphOptions.ext]
@@ -122,30 +170,138 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph {
public:
absl::StatusOr<CalculatorGraphConfig> GetConfig(
SubgraphContext* sc) override {
ASSIGN_OR_RETURN(
const auto* model_resources,
CreateModelResources<HandGestureRecognizerGraphOptions>(sc));
if (sc->Options<HandGestureRecognizerGraphOptions>()
.base_options()
.has_model_asset()) {
ASSIGN_OR_RETURN(
const auto* model_asset_bundle_resources,
CreateModelAssetBundleResources<HandGestureRecognizerGraphOptions>(
sc));
// When the model resources cache service is available, filling in
// the file pointer meta in the subtasks' base options. Otherwise,
// providing the file contents instead.
MP_RETURN_IF_ERROR(SetSubTaskBaseOptions(
*model_asset_bundle_resources,
sc->MutableOptions<HandGestureRecognizerGraphOptions>(),
!sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService)
.IsAvailable()));
}
ASSIGN_OR_RETURN(const auto sub_task_model_resources,
CreateSubTaskModelResources(sc));
Graph graph;
ASSIGN_OR_RETURN(
auto hand_gestures,
BuildGestureRecognizerGraph(
sc->Options<HandGestureRecognizerGraphOptions>(), *model_resources,
graph[Input<ClassificationList>(kHandednessTag)],
graph[Input<NormalizedLandmarkList>(kLandmarksTag)],
graph[Input<LandmarkList>(kWorldLandmarksTag)],
graph[Input<std::pair<int, int>>(kImageSizeTag)], graph));
ASSIGN_OR_RETURN(auto hand_gestures,
BuildGestureRecognizerGraph(
sc->Options<HandGestureRecognizerGraphOptions>(),
sub_task_model_resources,
graph[Input<ClassificationList>(kHandednessTag)],
graph[Input<NormalizedLandmarkList>(kLandmarksTag)],
graph[Input<LandmarkList>(kWorldLandmarksTag)],
graph[Input<std::pair<int, int>>(kImageSizeTag)],
graph[Input<NormalizedRect>(kNormRectTag)], graph));
hand_gestures >> graph[Output<ClassificationList>(kHandGesturesTag)];
return graph.GetConfig();
}
private:
// Sets the base options in the sub tasks.
absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
HandGestureRecognizerGraphOptions* options,
bool is_copy) {
ASSIGN_OR_RETURN(const auto gesture_embedder_file,
resources.GetModelFile(kGestureEmbedderTFLiteName));
auto* gesture_embedder_graph_options =
options->mutable_gesture_embedder_graph_options();
SetExternalFile(gesture_embedder_file,
gesture_embedder_graph_options->mutable_base_options()
->mutable_model_asset(),
is_copy);
PopulateAccelerationAndUseStreamMode(
options->base_options(),
gesture_embedder_graph_options->mutable_base_options());
ASSIGN_OR_RETURN(
const auto canned_gesture_classifier_file,
resources.GetModelFile(kCannedGestureClassifierTFLiteName));
auto* canned_gesture_classifier_graph_options =
options->mutable_canned_gesture_classifier_graph_options();
SetExternalFile(
canned_gesture_classifier_file,
canned_gesture_classifier_graph_options->mutable_base_options()
->mutable_model_asset(),
is_copy);
PopulateAccelerationAndUseStreamMode(
options->base_options(),
canned_gesture_classifier_graph_options->mutable_base_options());
const auto custom_gesture_classifier_file =
resources.GetModelFile(kCustomGestureClassifierTFLiteName);
if (custom_gesture_classifier_file.ok()) {
has_custom_gesture_classifier = true;
auto* custom_gesture_classifier_graph_options =
options->mutable_custom_gesture_classifier_graph_options();
SetExternalFile(
custom_gesture_classifier_file.value(),
custom_gesture_classifier_graph_options->mutable_base_options()
->mutable_model_asset(),
is_copy);
PopulateAccelerationAndUseStreamMode(
options->base_options(),
custom_gesture_classifier_graph_options->mutable_base_options());
} else {
LOG(INFO) << "Custom gesture classifier is not defined.";
}
return absl::OkStatus();
}
absl::StatusOr<SubTaskModelResources> CreateSubTaskModelResources(
SubgraphContext* sc) {
auto* options = sc->MutableOptions<HandGestureRecognizerGraphOptions>();
SubTaskModelResources sub_task_model_resources;
auto& gesture_embedder_model_asset =
*options->mutable_gesture_embedder_graph_options()
->mutable_base_options()
->mutable_model_asset();
ASSIGN_OR_RETURN(
sub_task_model_resources.gesture_embedder_model_resource,
CreateModelResources(sc,
std::make_unique<core::proto::ExternalFile>(
std::move(gesture_embedder_model_asset)),
"_gesture_embedder"));
auto& canned_gesture_classifier_model_asset =
*options->mutable_canned_gesture_classifier_graph_options()
->mutable_base_options()
->mutable_model_asset();
ASSIGN_OR_RETURN(
sub_task_model_resources.canned_gesture_classifier_model_resource,
CreateModelResources(
sc,
std::make_unique<core::proto::ExternalFile>(
std::move(canned_gesture_classifier_model_asset)),
"_canned_gesture_classifier"));
if (has_custom_gesture_classifier) {
auto& custom_gesture_classifier_model_asset =
*options->mutable_custom_gesture_classifier_graph_options()
->mutable_base_options()
->mutable_model_asset();
ASSIGN_OR_RETURN(
sub_task_model_resources.custom_gesture_classifier_model_resource,
CreateModelResources(
sc,
std::make_unique<core::proto::ExternalFile>(
std::move(custom_gesture_classifier_model_asset)),
"_custom_gesture_classifier"));
}
return sub_task_model_resources;
}
absl::StatusOr<Source<ClassificationList>> BuildGestureRecognizerGraph(
const HandGestureRecognizerGraphOptions& graph_options,
const core::ModelResources& model_resources,
const SubTaskModelResources& sub_task_model_resources,
Source<ClassificationList> handedness,
Source<NormalizedLandmarkList> hand_landmarks,
Source<LandmarkList> hand_world_landmarks,
Source<std::pair<int, int>> image_size, Graph& graph) {
Source<std::pair<int, int>> image_size, Source<NormalizedRect> norm_rect,
Graph& graph) {
// Converts the ClassificationList to a matrix.
auto& handedness_to_matrix = graph.AddNode("HandednessToMatrixCalculator");
handedness >> handedness_to_matrix.In(kHandednessTag);
@@ -166,6 +322,7 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph {
landmarks_options;
hand_landmarks >> hand_landmarks_to_matrix.In(kLandmarksTag);
image_size >> hand_landmarks_to_matrix.In(kImageSizeTag);
norm_rect >> hand_landmarks_to_matrix.In(kNormRectTag);
auto hand_landmarks_matrix =
hand_landmarks_to_matrix[Output<Matrix>(kLandmarksMatrixTag)];
@@ -181,6 +338,7 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph {
hand_world_landmarks >>
hand_world_landmarks_to_matrix.In(kWorldLandmarksTag);
image_size >> hand_world_landmarks_to_matrix.In(kImageSizeTag);
norm_rect >> hand_world_landmarks_to_matrix.In(kNormRectTag);
auto hand_world_landmarks_matrix =
hand_world_landmarks_to_matrix[Output<Matrix>(kLandmarksMatrixTag)];
@@ -198,26 +356,71 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph {
hand_world_landmarks_tensor >> concatenate_tensor_vector.In(2);
auto concatenated_tensors = concatenate_tensor_vector.Out("");
// Inference for static hand gesture recognition.
// TODO add embedding step.
auto& inference = AddInference(
model_resources, graph_options.base_options().acceleration(), graph);
concatenated_tensors >> inference.In(kTensorsTag);
auto inference_output_tensors = inference.Out(kTensorsTag);
// Inference for gesture embedder.
auto& gesture_embedder_inference =
AddInference(*sub_task_model_resources.gesture_embedder_model_resource,
graph_options.gesture_embedder_graph_options()
.base_options()
.acceleration(),
graph);
concatenated_tensors >> gesture_embedder_inference.In(kTensorsTag);
auto embedding_tensors =
gesture_embedder_inference.Out(kTensorsTag).Cast<Tensor>();
auto& combine_predictions = graph.AddNode("CombinedPredictionCalculator");
MP_RETURN_IF_ERROR(ConfigureCombinedPredictionCalculator(
&combine_predictions
.GetOptions<CombinedPredictionCalculatorOptions>()));
int classifier_nums = 0;
// Inference for custom gesture classifier if it exists.
if (has_custom_gesture_classifier) {
ASSIGN_OR_RETURN(
auto gesture_clasification_list,
GetGestureClassificationList(
sub_task_model_resources.custom_gesture_classifier_model_resource,
graph_options.custom_gesture_classifier_graph_options(),
embedding_tensors, graph));
gesture_clasification_list >> combine_predictions.In(classifier_nums++);
}
// Inference for canned gesture classifier.
ASSIGN_OR_RETURN(
auto gesture_clasification_list,
GetGestureClassificationList(
sub_task_model_resources.canned_gesture_classifier_model_resource,
graph_options.canned_gesture_classifier_graph_options(),
embedding_tensors, graph));
gesture_clasification_list >> combine_predictions.In(classifier_nums++);
auto combined_classification_list =
combine_predictions.Out(kPredictionTag).Cast<ClassificationList>();
return combined_classification_list;
}
absl::StatusOr<Source<ClassificationList>> GetGestureClassificationList(
const core::ModelResources* model_resources,
const proto::GestureClassifierGraphOptions& options,
Source<Tensor>& embedding_tensors, Graph& graph) {
auto& gesture_classifier_inference = AddInference(
*model_resources, options.base_options().acceleration(), graph);
embedding_tensors >> gesture_classifier_inference.In(kTensorsTag);
auto gesture_inference_out_tensors =
gesture_classifier_inference.Out(kTensorsTag);
auto& tensors_to_classification =
graph.AddNode("TensorsToClassificationCalculator");
MP_RETURN_IF_ERROR(ConfigureTensorsToClassificationCalculator(
graph_options.classifier_options(),
*model_resources.GetMetadataExtractor(), 0,
options.classifier_options(), *model_resources->GetMetadataExtractor(),
0,
&tensors_to_classification.GetOptions<
mediapipe::TensorsToClassificationCalculatorOptions>()));
inference_output_tensors >> tensors_to_classification.In(kTensorsTag);
auto classification_list =
tensors_to_classification[Output<ClassificationList>(
"CLASSIFICATIONS")];
return classification_list;
gesture_inference_out_tensors >> tensors_to_classification.In(kTensorsTag);
return tensors_to_classification.Out("CLASSIFICATIONS")
.Cast<ClassificationList>();
}
bool has_custom_gesture_classifier = false;
};
// clang-format off
@@ -239,6 +442,9 @@ REGISTER_MEDIAPIPE_GRAPH(
// A vector hand landmarks in world coordinates.
// IMAGE_SIZE - std::pair<int, int>
// The size of image from which the landmarks detected from.
// NORM_RECT - NormalizedRect
// NormalizedRect whose 'rotation' field is used to rotate the
// landmarks before processing them.
// HAND_TRACKING_IDS - std::vector<int>
// A vector of the tracking ids of the hands. The tracking id is the vector
// index corresponding to the same hand if the graph runs multiple times.
@@ -257,6 +463,7 @@ REGISTER_MEDIAPIPE_GRAPH(
// input_stream: "LANDMARKS:landmarks"
// input_stream: "WORLD_LANDMARKS:world_landmarks"
// input_stream: "IMAGE_SIZE:image_size"
// input_stream: "NORM_RECT:norm_rect"
// input_stream: "HAND_TRACKING_IDS:hand_tracking_ids"
// output_stream: "HAND_GESTURES:hand_gestures"
// options {
@@ -283,6 +490,7 @@ class MultipleHandGestureRecognizerGraph : public core::ModelTaskGraph {
graph[Input<std::vector<NormalizedLandmarkList>>(kLandmarksTag)],
graph[Input<std::vector<LandmarkList>>(kWorldLandmarksTag)],
graph[Input<std::pair<int, int>>(kImageSizeTag)],
graph[Input<NormalizedRect>(kNormRectTag)],
graph[Input<std::vector<int>>(kHandTrackingIdsTag)], graph));
multi_hand_gestures >>
graph[Output<std::vector<ClassificationList>>(kHandGesturesTag)];
@@ -296,18 +504,20 @@ class MultipleHandGestureRecognizerGraph : public core::ModelTaskGraph {
Source<std::vector<ClassificationList>> multi_handedness,
Source<std::vector<NormalizedLandmarkList>> multi_hand_landmarks,
Source<std::vector<LandmarkList>> multi_hand_world_landmarks,
Source<std::pair<int, int>> image_size,
Source<std::pair<int, int>> image_size, Source<NormalizedRect> norm_rect,
Source<std::vector<int>> multi_hand_tracking_ids, Graph& graph) {
auto& begin_loop_int = graph.AddNode("BeginLoopIntCalculator");
image_size >> begin_loop_int.In(kCloneTag)[0];
multi_handedness >> begin_loop_int.In(kCloneTag)[1];
multi_hand_landmarks >> begin_loop_int.In(kCloneTag)[2];
multi_hand_world_landmarks >> begin_loop_int.In(kCloneTag)[3];
norm_rect >> begin_loop_int.In(kCloneTag)[1];
multi_handedness >> begin_loop_int.In(kCloneTag)[2];
multi_hand_landmarks >> begin_loop_int.In(kCloneTag)[3];
multi_hand_world_landmarks >> begin_loop_int.In(kCloneTag)[4];
multi_hand_tracking_ids >> begin_loop_int.In(kIterableTag);
auto image_size_clone = begin_loop_int.Out(kCloneTag)[0];
auto multi_handedness_clone = begin_loop_int.Out(kCloneTag)[1];
auto multi_hand_landmarks_clone = begin_loop_int.Out(kCloneTag)[2];
auto multi_hand_world_landmarks_clone = begin_loop_int.Out(kCloneTag)[3];
auto norm_rect_clone = begin_loop_int.Out(kCloneTag)[1];
auto multi_handedness_clone = begin_loop_int.Out(kCloneTag)[2];
auto multi_hand_landmarks_clone = begin_loop_int.Out(kCloneTag)[3];
auto multi_hand_world_landmarks_clone = begin_loop_int.Out(kCloneTag)[4];
auto hand_tracking_id = begin_loop_int.Out(kItemTag);
auto batch_end = begin_loop_int.Out(kBatchEndTag);
@@ -341,6 +551,7 @@ class MultipleHandGestureRecognizerGraph : public core::ModelTaskGraph {
hand_world_landmarks >>
hand_gesture_recognizer_graph.In(kWorldLandmarksTag);
image_size_clone >> hand_gesture_recognizer_graph.In(kImageSizeTag);
norm_rect_clone >> hand_gesture_recognizer_graph.In(kNormRectTag);
auto hand_gestures = hand_gesture_recognizer_graph.Out(kHandGesturesTag);
auto& end_loop_classification_lists =
mediapipe/tasks/cc/vision/gesture_recognizer/proto/BUILD
-1
View File
@@ -49,7 +49,6 @@ mediapipe_proto_library(
":gesture_embedder_graph_options_proto",
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_proto",
"//mediapipe/tasks/cc/core/proto:base_options_proto",
],
)
mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.proto
+1 -6
View File
@@ -18,7 +18,6 @@ syntax = "proto2";
package mediapipe.tasks.vision.gesture_recognizer.proto;
import "mediapipe/framework/calculator.proto";
import "mediapipe/tasks/cc/components/processors/proto/classifier_options.proto";
import "mediapipe/tasks/cc/core/proto/base_options.proto";
import "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_classifier_graph_options.proto";
import "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_embedder_graph_options.proto";
@@ -37,15 +36,11 @@ message HandGestureRecognizerGraphOptions {
// Options for GestureEmbedder.
optional GestureEmbedderGraphOptions gesture_embedder_graph_options = 2;
// Options for GestureClassifier of default gestures.
// Options for GestureClassifier of canned gestures.
optional GestureClassifierGraphOptions
canned_gesture_classifier_graph_options = 3;
// Options for GestureClassifier of custom gestures.
optional GestureClassifierGraphOptions
custom_gesture_classifier_graph_options = 4;
// TODO: remove these. Temporary solutions before bundle asset is
// ready.
optional components.processors.proto.ClassifierOptions classifier_options = 5;
}
mediapipe/tasks/cc/vision/hand_detector/BUILD
+1 -1
View File
@@ -32,7 +32,7 @@ cc_library(
"//mediapipe/calculators/tflite:ssd_anchors_calculator_cc_proto",
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
"//mediapipe/calculators/util:detection_label_id_to_text_calculator_cc_proto",
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
"//mediapipe/calculators/util:detection_projection_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator_cc_proto",
"//mediapipe/calculators/util:non_max_suppression_calculator",
mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph.cc
+23 -18
View File
@@ -58,6 +58,7 @@ using ::mediapipe::tasks::vision::hand_detector::proto::
HandDetectorGraphOptions;
constexpr char kImageTag[] = "IMAGE";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kPalmDetectionsTag[] = "PALM_DETECTIONS";
constexpr char kHandRectsTag[] = "HAND_RECTS";
constexpr char kPalmRectsTag[] = "PALM_RECTS";
@@ -148,6 +149,9 @@ void ConfigureRectTransformationCalculator(
// Inputs:
// IMAGE - Image
// Image to perform detection on.
// NORM_RECT - NormalizedRect
// Describes image rotation and region of image to perform detection
// on.
//
// Outputs:
// PALM_DETECTIONS - std::vector<Detection>
@@ -159,11 +163,14 @@ void ConfigureRectTransformationCalculator(
// IMAGE - Image
// The input image that the hand detector runs on and has the pixel data
// stored on the target storage (CPU vs GPU).
// All returned coordinates are in the unrotated and uncropped input image
// coordinates system.
//
// Example:
// node {
// calculator: "mediapipe.tasks.vision.hand_detector.HandDetectorGraph"
// input_stream: "IMAGE:image"
// input_stream: "NORM_RECT:norm_rect"
// output_stream: "PALM_DETECTIONS:palm_detections"
// output_stream: "HAND_RECTS:hand_rects_from_palm_detections"
// output_stream: "PALM_RECTS:palm_rects"
@@ -189,11 +196,11 @@ class HandDetectorGraph : public core::ModelTaskGraph {
ASSIGN_OR_RETURN(const auto* model_resources,
CreateModelResources<HandDetectorGraphOptions>(sc));
Graph graph;
ASSIGN_OR_RETURN(
auto hand_detection_outs,
BuildHandDetectionSubgraph(sc->Options<HandDetectorGraphOptions>(),
*model_resources,
graph[Input<Image>(kImageTag)], graph));
ASSIGN_OR_RETURN(auto hand_detection_outs,
BuildHandDetectionSubgraph(
sc->Options<HandDetectorGraphOptions>(),
*model_resources, graph[Input<Image>(kImageTag)],
graph[Input<NormalizedRect>(kNormRectTag)], graph));
hand_detection_outs.palm_detections >>
graph[Output<std::vector<Detection>>(kPalmDetectionsTag)];
hand_detection_outs.hand_rects >>
@@ -216,7 +223,7 @@ class HandDetectorGraph : public core::ModelTaskGraph {
absl::StatusOr<HandDetectionOuts> BuildHandDetectionSubgraph(
const HandDetectorGraphOptions& subgraph_options,
const core::ModelResources& model_resources, Source<Image> image_in,
Graph& graph) {
Source<NormalizedRect> norm_rect_in, Graph& graph) {
// Add image preprocessing subgraph. The model expects aspect ratio
// unchanged.
auto& preprocessing =
@@ -228,13 +235,16 @@ class HandDetectorGraph : public core::ModelTaskGraph {
image_to_tensor_options.set_keep_aspect_ratio(true);
image_to_tensor_options.set_border_mode(
mediapipe::ImageToTensorCalculatorOptions::BORDER_ZERO);
bool use_gpu = components::DetermineImagePreprocessingGpuBackend(
subgraph_options.base_options().acceleration());
MP_RETURN_IF_ERROR(ConfigureImagePreprocessing(
model_resources,
model_resources, use_gpu,
&preprocessing
.GetOptions<tasks::components::ImagePreprocessingOptions>()));
image_in >> preprocessing.In("IMAGE");
norm_rect_in >> preprocessing.In("NORM_RECT");
auto preprocessed_tensors = preprocessing.Out("TENSORS");
auto letterbox_padding = preprocessing.Out("LETTERBOX_PADDING");
auto matrix = preprocessing.Out("MATRIX");
auto image_size = preprocessing.Out("IMAGE_SIZE");
// Adds SSD palm detection model.
@@ -278,17 +288,12 @@ class HandDetectorGraph : public core::ModelTaskGraph {
nms_detections >> detection_label_id_to_text.In("");
auto detections_with_text = detection_label_id_to_text.Out("");
// Adjusts detection locations (already normalized to [0.f, 1.f]) on the
// letterboxed image (after image transformation with the FIT scale mode) to
// the corresponding locations on the same image with the letterbox removed
// (the input image to the graph before image transformation).
auto& detection_letterbox_removal =
graph.AddNode("DetectionLetterboxRemovalCalculator");
detections_with_text >> detection_letterbox_removal.In("DETECTIONS");
letterbox_padding >> detection_letterbox_removal.In("LETTERBOX_PADDING");
// Projects detections back into the input image coordinates system.
auto& detection_projection = graph.AddNode("DetectionProjectionCalculator");
detections_with_text >> detection_projection.In("DETECTIONS");
matrix >> detection_projection.In("PROJECTION_MATRIX");
auto palm_detections =
detection_letterbox_removal[Output<std::vector<Detection>>(
"DETECTIONS")];
detection_projection[Output<std::vector<Detection>>("DETECTIONS")];
// Converts each palm detection into a rectangle (normalized by image size)
// that encloses the palm and is rotated such that the line connecting
mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph_test.cc
+31 -3
View File
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cmath>
#include <iostream>
#include <memory>
#include <string>
@@ -75,13 +76,18 @@ using ::testing::proto::Partially;
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
constexpr char kPalmDetectionModel[] = "palm_detection_full.tflite";
constexpr char kTestRightHandsImage[] = "right_hands.jpg";
constexpr char kTestRightHandsRotatedImage[] = "right_hands_rotated.jpg";
constexpr char kTestModelResourcesTag[] = "test_model_resources";
constexpr char kOneHandResultFile[] = "hand_detector_result_one_hand.pbtxt";
constexpr char kOneHandRotatedResultFile[] =
"hand_detector_result_one_hand_rotated.pbtxt";
constexpr char kTwoHandsResultFile[] = "hand_detector_result_two_hands.pbtxt";
constexpr char kImageTag[] = "IMAGE";
constexpr char kImageName[] = "image";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kNormRectName[] = "norm_rect";
constexpr char kPalmDetectionsTag[] = "PALM_DETECTIONS";
constexpr char kPalmDetectionsName[] = "palm_detections";
constexpr char kHandRectsTag[] = "HAND_RECTS";
@@ -117,6 +123,8 @@ absl::StatusOr<std::unique_ptr<TaskRunner>> CreateTaskRunner(
graph[Input<Image>(kImageTag)].SetName(kImageName) >>
hand_detection.In(kImageTag);
graph[Input<NormalizedRect>(kNormRectTag)].SetName(kNormRectName) >>
hand_detection.In(kNormRectTag);
hand_detection.Out(kPalmDetectionsTag).SetName(kPalmDetectionsName) >>
graph[Output<std::vector<Detection>>(kPalmDetectionsTag)];
@@ -142,6 +150,9 @@ struct TestParams {
std::string hand_detection_model_name;
// The filename of test image.
std::string test_image_name;
// The rotation to apply to the test image before processing, in radians
// counter-clockwise.
float rotation;
// The number of maximum detected hands.
int num_hands;
// The expected hand detector result.
@@ -154,14 +165,22 @@ TEST_P(HandDetectionTest, DetectTwoHands) {
MP_ASSERT_OK_AND_ASSIGN(
Image image, DecodeImageFromFile(JoinPath("./", kTestDataDirectory,
GetParam().test_image_name)));
NormalizedRect input_norm_rect;
input_norm_rect.set_rotation(GetParam().rotation);
input_norm_rect.set_x_center(0.5);
input_norm_rect.set_y_center(0.5);
input_norm_rect.set_width(1.0);
input_norm_rect.set_height(1.0);
MP_ASSERT_OK_AND_ASSIGN(
auto model_resources,
CreateModelResourcesForModel(GetParam().hand_detection_model_name));
MP_ASSERT_OK_AND_ASSIGN(
auto task_runner, CreateTaskRunner(*model_resources, kPalmDetectionModel,
GetParam().num_hands));
auto output_packets =
task_runner->Process({{kImageName, MakePacket<Image>(std::move(image))}});
auto output_packets = task_runner->Process(
{{kImageName, MakePacket<Image>(std::move(image))},
{kNormRectName,
MakePacket<NormalizedRect>(std::move(input_norm_rect))}});
MP_ASSERT_OK(output_packets);
const std::vector<Detection>& palm_detections =
(*output_packets)[kPalmDetectionsName].Get<std::vector<Detection>>();
@@ -188,15 +207,24 @@ INSTANTIATE_TEST_SUITE_P(
Values(TestParams{.test_name = "DetectOneHand",
.hand_detection_model_name = kPalmDetectionModel,
.test_image_name = kTestRightHandsImage,
.rotation = 0,
.num_hands = 1,
.expected_result =
GetExpectedHandDetectorResult(kOneHandResultFile)},
TestParams{.test_name = "DetectTwoHands",
.hand_detection_model_name = kPalmDetectionModel,
.test_image_name = kTestRightHandsImage,
.rotation = 0,
.num_hands = 2,
.expected_result =
GetExpectedHandDetectorResult(kTwoHandsResultFile)}),
GetExpectedHandDetectorResult(kTwoHandsResultFile)},
TestParams{.test_name = "DetectOneHandWithRotation",
.hand_detection_model_name = kPalmDetectionModel,
.test_image_name = kTestRightHandsRotatedImage,
.rotation = M_PI / 2.0f,
.num_hands = 1,
.expected_result = GetExpectedHandDetectorResult(
kOneHandRotatedResultFile)}),
[](const TestParamInfo<HandDetectionTest::ParamType>& info) {
return info.param.test_name;
});
mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc
+32 -9
View File
@@ -64,6 +64,7 @@ using ::mediapipe::tasks::vision::hand_landmarker::proto::
HandLandmarksDetectorGraphOptions;
constexpr char kImageTag[] = "IMAGE";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kWorldLandmarksTag[] = "WORLD_LANDMARKS";
constexpr char kHandRectNextFrameTag[] = "HAND_RECT_NEXT_FRAME";
@@ -91,18 +92,30 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
bool is_copy) {
ASSIGN_OR_RETURN(const auto hand_detector_file,
resources.GetModelFile(kHandDetectorTFLiteName));
auto* hand_detector_graph_options =
options->mutable_hand_detector_graph_options();
SetExternalFile(hand_detector_file,
options->mutable_hand_detector_graph_options()
->mutable_base_options()
hand_detector_graph_options->mutable_base_options()
->mutable_model_asset(),
is_copy);
hand_detector_graph_options->mutable_base_options()
->mutable_acceleration()
->CopyFrom(options->base_options().acceleration());
hand_detector_graph_options->mutable_base_options()->set_use_stream_mode(
options->base_options().use_stream_mode());
ASSIGN_OR_RETURN(const auto hand_landmarks_detector_file,
resources.GetModelFile(kHandLandmarksDetectorTFLiteName));
auto* hand_landmarks_detector_graph_options =
options->mutable_hand_landmarks_detector_graph_options();
SetExternalFile(hand_landmarks_detector_file,
options->mutable_hand_landmarks_detector_graph_options()
->mutable_base_options()
hand_landmarks_detector_graph_options->mutable_base_options()
->mutable_model_asset(),
is_copy);
hand_landmarks_detector_graph_options->mutable_base_options()
->mutable_acceleration()
->CopyFrom(options->base_options().acceleration());
hand_landmarks_detector_graph_options->mutable_base_options()
->set_use_stream_mode(options->base_options().use_stream_mode());
return absl::OkStatus();
}
@@ -122,6 +135,9 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
// Inputs:
// IMAGE - Image
// Image to perform hand landmarks detection on.
// NORM_RECT - NormalizedRect
// Describes image rotation and region of image to perform landmarks
// detection on.
//
// Outputs:
// LANDMARKS: - std::vector<NormalizedLandmarkList>
@@ -140,11 +156,14 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
// IMAGE - Image
// The input image that the hand landmarker runs on and has the pixel data
// stored on the target storage (CPU vs GPU).
// All returned coordinates are in the unrotated and uncropped input image
// coordinates system.
//
// Example:
// node {
// calculator: "mediapipe.tasks.vision.hand_landmarker.HandLandmarkerGraph"
// input_stream: "IMAGE:image_in"
// input_stream: "NORM_RECT:norm_rect"
// output_stream: "LANDMARKS:hand_landmarks"
// output_stream: "WORLD_LANDMARKS:world_hand_landmarks"
// output_stream: "HAND_RECT_NEXT_FRAME:hand_rect_next_frame"
@@ -198,10 +217,11 @@ class HandLandmarkerGraph : public core::ModelTaskGraph {
!sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService)
.IsAvailable()));
}
ASSIGN_OR_RETURN(
auto hand_landmarker_outputs,
BuildHandLandmarkerGraph(sc->Options<HandLandmarkerGraphOptions>(),
graph[Input<Image>(kImageTag)], graph));
ASSIGN_OR_RETURN(auto hand_landmarker_outputs,
BuildHandLandmarkerGraph(
sc->Options<HandLandmarkerGraphOptions>(),
graph[Input<Image>(kImageTag)],
graph[Input<NormalizedRect>(kNormRectTag)], graph));
hand_landmarker_outputs.landmark_lists >>
graph[Output<std::vector<NormalizedLandmarkList>>(kLandmarksTag)];
hand_landmarker_outputs.world_landmark_lists >>
@@ -240,7 +260,7 @@ class HandLandmarkerGraph : public core::ModelTaskGraph {
// graph: the mediapipe graph instance to be updated.
absl::StatusOr<HandLandmarkerOutputs> BuildHandLandmarkerGraph(
const HandLandmarkerGraphOptions& tasks_options, Source<Image> image_in,
Graph& graph) {
Source<NormalizedRect> norm_rect_in, Graph& graph) {
const int max_num_hands =
tasks_options.hand_detector_graph_options().num_hands();
@@ -258,12 +278,15 @@ class HandLandmarkerGraph : public core::ModelTaskGraph {
auto image_for_hand_detector =
DisallowIf(image_in, has_enough_hands, graph);
auto norm_rect_in_for_hand_detector =
DisallowIf(norm_rect_in, has_enough_hands, graph);
auto& hand_detector =
graph.AddNode("mediapipe.tasks.vision.hand_detector.HandDetectorGraph");
hand_detector.GetOptions<HandDetectorGraphOptions>().CopyFrom(
tasks_options.hand_detector_graph_options());
image_for_hand_detector >> hand_detector.In("IMAGE");
norm_rect_in_for_hand_detector >> hand_detector.In("NORM_RECT");
auto hand_rects_from_hand_detector = hand_detector.Out("HAND_RECTS");
auto& hand_association = graph.AddNode("HandAssociationCalculator");
mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph_test.cc
+54 -3
View File
@@ -13,10 +13,12 @@ See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cmath>
#include <iostream>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/flags/flag.h"
#include "absl/status/statusor.h"
@@ -65,11 +67,14 @@ using ::testing::proto::Approximately;
using ::testing::proto::Partially;
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
constexpr char kHandLandmarkerModelBundle[] = "hand_landmark.task";
constexpr char kHandLandmarkerModelBundle[] = "hand_landmarker.task";
constexpr char kLeftHandsImage[] = "left_hands.jpg";
constexpr char kLeftHandsRotatedImage[] = "left_hands_rotated.jpg";
constexpr char kImageTag[] = "IMAGE";
constexpr char kImageName[] = "image_in";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kNormRectName[] = "norm_rect_in";
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kLandmarksName[] = "landmarks";
constexpr char kWorldLandmarksTag[] = "WORLD_LANDMARKS";
@@ -84,6 +89,11 @@ constexpr char kExpectedLeftUpHandLandmarksFilename[] =
"expected_left_up_hand_landmarks.prototxt";
constexpr char kExpectedLeftDownHandLandmarksFilename[] =
"expected_left_down_hand_landmarks.prototxt";
// Same but for the rotated image.
constexpr char kExpectedLeftUpHandRotatedLandmarksFilename[] =
"expected_left_up_hand_rotated_landmarks.prototxt";
constexpr char kExpectedLeftDownHandRotatedLandmarksFilename[] =
"expected_left_down_hand_rotated_landmarks.prototxt";
constexpr float kFullModelFractionDiff = 0.03; // percentage
constexpr float kAbsMargin = 0.03;
@@ -111,6 +121,8 @@ absl::StatusOr<std::unique_ptr<TaskRunner>> CreateTaskRunner() {
graph[Input<Image>(kImageTag)].SetName(kImageName) >>
hand_landmarker_graph.In(kImageTag);
graph[Input<NormalizedRect>(kNormRectTag)].SetName(kNormRectName) >>
hand_landmarker_graph.In(kNormRectTag);
hand_landmarker_graph.Out(kLandmarksTag).SetName(kLandmarksName) >>
graph[Output<std::vector<NormalizedLandmarkList>>(kLandmarksTag)];
hand_landmarker_graph.Out(kWorldLandmarksTag).SetName(kWorldLandmarksName) >>
@@ -130,9 +142,16 @@ TEST_F(HandLandmarkerTest, Succeeds) {
MP_ASSERT_OK_AND_ASSIGN(
Image image,
DecodeImageFromFile(JoinPath("./", kTestDataDirectory, kLeftHandsImage)));
NormalizedRect input_norm_rect;
input_norm_rect.set_x_center(0.5);
input_norm_rect.set_y_center(0.5);
input_norm_rect.set_width(1.0);
input_norm_rect.set_height(1.0);
MP_ASSERT_OK_AND_ASSIGN(auto task_runner, CreateTaskRunner());
auto output_packets =
task_runner->Process({{kImageName, MakePacket<Image>(std::move(image))}});
auto output_packets = task_runner->Process(
{{kImageName, MakePacket<Image>(std::move(image))},
{kNormRectName,
MakePacket<NormalizedRect>(std::move(input_norm_rect))}});
const auto& landmarks = (*output_packets)[kLandmarksName]
.Get<std::vector<NormalizedLandmarkList>>();
ASSERT_EQ(landmarks.size(), kMaxNumHands);
@@ -150,6 +169,38 @@ TEST_F(HandLandmarkerTest, Succeeds) {
/*fraction=*/kFullModelFractionDiff));
}
TEST_F(HandLandmarkerTest, SucceedsWithRotation) {
MP_ASSERT_OK_AND_ASSIGN(
Image image, DecodeImageFromFile(JoinPath("./", kTestDataDirectory,
kLeftHandsRotatedImage)));
NormalizedRect input_norm_rect;
input_norm_rect.set_x_center(0.5);
input_norm_rect.set_y_center(0.5);
input_norm_rect.set_width(1.0);
input_norm_rect.set_height(1.0);
input_norm_rect.set_rotation(M_PI / 2.0);
MP_ASSERT_OK_AND_ASSIGN(auto task_runner, CreateTaskRunner());
auto output_packets = task_runner->Process(
{{kImageName, MakePacket<Image>(std::move(image))},
{kNormRectName,
MakePacket<NormalizedRect>(std::move(input_norm_rect))}});
const auto& landmarks = (*output_packets)[kLandmarksName]
.Get<std::vector<NormalizedLandmarkList>>();
ASSERT_EQ(landmarks.size(), kMaxNumHands);
std::vector<NormalizedLandmarkList> expected_landmarks = {
GetExpectedLandmarkList(kExpectedLeftUpHandRotatedLandmarksFilename),
GetExpectedLandmarkList(kExpectedLeftDownHandRotatedLandmarksFilename)};
EXPECT_THAT(landmarks[0],
Approximately(Partially(EqualsProto(expected_landmarks[0])),
/*margin=*/kAbsMargin,
/*fraction=*/kFullModelFractionDiff));
EXPECT_THAT(landmarks[1],
Approximately(Partially(EqualsProto(expected_landmarks[1])),
/*margin=*/kAbsMargin,
/*fraction=*/kFullModelFractionDiff));
}
} // namespace
} // namespace hand_landmarker
mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc
+3 -1
View File
@@ -283,8 +283,10 @@ class SingleHandLandmarksDetectorGraph : public core::ModelTaskGraph {
auto& preprocessing =
graph.AddNode("mediapipe.tasks.components.ImagePreprocessingSubgraph");
bool use_gpu = components::DetermineImagePreprocessingGpuBackend(
subgraph_options.base_options().acceleration());
MP_RETURN_IF_ERROR(ConfigureImagePreprocessing(
model_resources,
model_resources, use_gpu,
&preprocessing
.GetOptions<tasks::components::ImagePreprocessingOptions>()));
image_in >> preprocessing.In("IMAGE");
mediapipe/tasks/cc/vision/image_classifier/BUILD
+2
View File
@@ -50,6 +50,7 @@ cc_library(
"//mediapipe/framework/api2:builder",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/tasks/cc/components/containers:classification_result",
"//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto",
"//mediapipe/tasks/cc/components/processors:classifier_options",
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto",
@@ -59,6 +60,7 @@ cc_library(
"//mediapipe/tasks/cc/core/proto:base_options_cc_proto",
"//mediapipe/tasks/cc/core/proto:inference_subgraph_cc_proto",
"//mediapipe/tasks/cc/vision/core:base_vision_task_api",
"//mediapipe/tasks/cc/vision/core:image_processing_options",
"//mediapipe/tasks/cc/vision/core:running_mode",
"//mediapipe/tasks/cc/vision/core:vision_task_api_factory",
"//mediapipe/tasks/cc/vision/image_classifier/proto:image_classifier_graph_options_cc_proto",
mediapipe/tasks/cc/vision/image_classifier/image_classifier.cc
+30 -44
View File
@@ -26,6 +26,7 @@ limitations under the License.
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/tasks/cc/components/containers/classification_result.h"
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
#include "mediapipe/tasks/cc/components/processors/classifier_options.h"
#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h"
@@ -34,6 +35,7 @@ limitations under the License.
#include "mediapipe/tasks/cc/core/proto/inference_subgraph.pb.h"
#include "mediapipe/tasks/cc/core/task_runner.h"
#include "mediapipe/tasks/cc/core/utils.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
#include "mediapipe/tasks/cc/vision/core/vision_task_api_factory.h"
#include "mediapipe/tasks/cc/vision/image_classifier/proto/image_classifier_graph_options.pb.h"
@@ -45,8 +47,8 @@ namespace image_classifier {
namespace {
constexpr char kClassificationResultStreamName[] = "classification_result_out";
constexpr char kClassificationResultTag[] = "CLASSIFICATION_RESULT";
constexpr char kClassificationsStreamName[] = "classifications_out";
constexpr char kClassificationsTag[] = "CLASSIFICATIONS";
constexpr char kImageInStreamName[] = "image_in";
constexpr char kImageOutStreamName[] = "image_out";
constexpr char kImageTag[] = "IMAGE";
@@ -56,29 +58,10 @@ constexpr char kSubgraphTypeName[] =
"mediapipe.tasks.vision.image_classifier.ImageClassifierGraph";
constexpr int kMicroSecondsPerMilliSecond = 1000;
using ::mediapipe::tasks::components::containers::ConvertToClassificationResult;
using ::mediapipe::tasks::components::containers::proto::ClassificationResult;
using ::mediapipe::tasks::core::PacketMap;
// Returns a NormalizedRect covering the full image if input is not present.
// Otherwise, makes sure the x_center, y_center, width and height are set in
// case only a rotation was provided in the input.
NormalizedRect FillNormalizedRect(
std::optional<NormalizedRect> normalized_rect) {
NormalizedRect result;
if (normalized_rect.has_value()) {
result = *normalized_rect;
}
bool has_coordinates = result.has_x_center() || result.has_y_center() ||
result.has_width() || result.has_height();
if (!has_coordinates) {
result.set_x_center(0.5);
result.set_y_center(0.5);
result.set_width(1);
result.set_height(1);
}
return result;
}
// Creates a MediaPipe graph config that contains a subgraph node of
// type "ImageClassifierGraph". If the task is running in the live stream mode,
// a "FlowLimiterCalculator" will be added to limit the number of frames in
@@ -92,15 +75,13 @@ CalculatorGraphConfig CreateGraphConfig(
auto& task_subgraph = graph.AddNode(kSubgraphTypeName);
task_subgraph.GetOptions<proto::ImageClassifierGraphOptions>().Swap(
options_proto.get());
task_subgraph.Out(kClassificationResultTag)
.SetName(kClassificationResultStreamName) >>
graph.Out(kClassificationResultTag);
task_subgraph.Out(kClassificationsTag).SetName(kClassificationsStreamName) >>
graph.Out(kClassificationsTag);
task_subgraph.Out(kImageTag).SetName(kImageOutStreamName) >>
graph.Out(kImageTag);
if (enable_flow_limiting) {
return tasks::core::AddFlowLimiterCalculator(graph, task_subgraph,
{kImageTag, kNormRectTag},
kClassificationResultTag);
return tasks::core::AddFlowLimiterCalculator(
graph, task_subgraph, {kImageTag, kNormRectTag}, kClassificationsTag);
}
graph.In(kImageTag) >> task_subgraph.In(kImageTag);
graph.In(kNormRectTag) >> task_subgraph.In(kNormRectTag);
@@ -144,13 +125,14 @@ absl::StatusOr<std::unique_ptr<ImageClassifier>> ImageClassifier::Create(
if (status_or_packets.value()[kImageOutStreamName].IsEmpty()) {
return;
}
Packet classification_result_packet =
status_or_packets.value()[kClassificationResultStreamName];
Packet classifications_packet =
status_or_packets.value()[kClassificationsStreamName];
Packet image_packet = status_or_packets.value()[kImageOutStreamName];
result_callback(
classification_result_packet.Get<ClassificationResult>(),
ConvertToClassificationResult(
classifications_packet.Get<ClassificationResult>()),
image_packet.Get<Image>(),
classification_result_packet.Timestamp().Value() /
classifications_packet.Timestamp().Value() /
kMicroSecondsPerMilliSecond);
};
}
@@ -163,34 +145,37 @@ absl::StatusOr<std::unique_ptr<ImageClassifier>> ImageClassifier::Create(
std::move(packets_callback));
}
absl::StatusOr<ClassificationResult> ImageClassifier::Classify(
Image image, std::optional<NormalizedRect> image_processing_options) {
absl::StatusOr<ImageClassifierResult> ImageClassifier::Classify(
Image image,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"GPU input images are currently not supported.",
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
NormalizedRect norm_rect = FillNormalizedRect(image_processing_options);
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
{{kImageInStreamName, MakePacket<Image>(std::move(image))},
{kNormRectName, MakePacket<NormalizedRect>(std::move(norm_rect))}}));
return output_packets[kClassificationResultStreamName]
.Get<ClassificationResult>();
return ConvertToClassificationResult(
output_packets[kClassificationsStreamName].Get<ClassificationResult>());
}
absl::StatusOr<ClassificationResult> ImageClassifier::ClassifyForVideo(
absl::StatusOr<ImageClassifierResult> ImageClassifier::ClassifyForVideo(
Image image, int64 timestamp_ms,
std::optional<NormalizedRect> image_processing_options) {
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"GPU input images are currently not supported.",
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
NormalizedRect norm_rect = FillNormalizedRect(image_processing_options);
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@@ -200,20 +185,21 @@ absl::StatusOr<ClassificationResult> ImageClassifier::ClassifyForVideo(
{kNormRectName,
MakePacket<NormalizedRect>(std::move(norm_rect))
.At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}}));
return output_packets[kClassificationResultStreamName]
.Get<ClassificationResult>();
return ConvertToClassificationResult(
output_packets[kClassificationsStreamName].Get<ClassificationResult>());
}
absl::Status ImageClassifier::ClassifyAsync(
Image image, int64 timestamp_ms,
std::optional<NormalizedRect> image_processing_options) {
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"GPU input images are currently not supported.",
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
NormalizedRect norm_rect = FillNormalizedRect(image_processing_options);
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))
mediapipe/tasks/cc/vision/image_classifier/image_classifier.h
+23 -29
View File
@@ -22,11 +22,11 @@ limitations under the License.
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
#include "mediapipe/tasks/cc/components/containers/classification_result.h"
#include "mediapipe/tasks/cc/components/processors/classifier_options.h"
#include "mediapipe/tasks/cc/core/base_options.h"
#include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
namespace mediapipe {
@@ -34,6 +34,10 @@ namespace tasks {
namespace vision {
namespace image_classifier {
// Alias the shared ClassificationResult struct as result type.
using ImageClassifierResult =
::mediapipe::tasks::components::containers::ClassificationResult;
// The options for configuring a Mediapipe image classifier task.
struct ImageClassifierOptions {
// Base options for configuring MediaPipe Tasks, such as specifying the model
@@ -56,9 +60,8 @@ struct ImageClassifierOptions {
// The user-defined result callback for processing live stream data.
// The result callback should only be specified when the running mode is set
// to RunningMode::LIVE_STREAM.
std::function<void(
absl::StatusOr<components::containers::proto::ClassificationResult>,
const Image&, int64)>
std::function<void(absl::StatusOr<ImageClassifierResult>, const Image&,
int64)>
result_callback = nullptr;
};
@@ -109,12 +112,10 @@ class ImageClassifier : tasks::vision::core::BaseVisionTaskApi {
//
// The optional 'image_processing_options' parameter can be used to specify:
// - the rotation to apply to the image before performing classification, by
// setting its 'rotation' field in radians (e.g. 'M_PI / 2' for a 90°
// anti-clockwise rotation).
// setting its 'rotation_degrees' field.
// and/or
// - the region-of-interest on which to perform classification, by setting its
// 'x_center', 'y_center', 'width' and 'height' fields. If none of these is
// set, they will automatically be set to cover the full image.
// 'region_of_interest' field. If not specified, the full image is used.
// If both are specified, the crop around the region-of-interest is extracted
// first, then the specified rotation is applied to the crop.
//
@@ -124,21 +125,19 @@ class ImageClassifier : tasks::vision::core::BaseVisionTaskApi {
// The image can be of any size with format RGB or RGBA.
// TODO: describe exact preprocessing steps once
// YUVToImageCalculator is integrated.
absl::StatusOr<components::containers::proto::ClassificationResult> Classify(
absl::StatusOr<ImageClassifierResult> Classify(
mediapipe::Image image,
std::optional<mediapipe::NormalizedRect> image_processing_options =
std::optional<core::ImageProcessingOptions> image_processing_options =
std::nullopt);
// Performs image classification on the provided video frame.
//
// The optional 'image_processing_options' parameter can be used to specify:
// - the rotation to apply to the image before performing classification, by
// setting its 'rotation' field in radians (e.g. 'M_PI / 2' for a 90°
// anti-clockwise rotation).
// setting its 'rotation_degrees' field.
// and/or
// - the region-of-interest on which to perform classification, by setting its
// 'x_center', 'y_center', 'width' and 'height' fields. If none of these is
// set, they will automatically be set to cover the full image.
// 'region_of_interest' field. If not specified, the full image is used.
// If both are specified, the crop around the region-of-interest is extracted
// first, then the specified rotation is applied to the crop.
//
@@ -148,22 +147,20 @@ class ImageClassifier : tasks::vision::core::BaseVisionTaskApi {
// The image can be of any size with format RGB or RGBA. It's required to
// provide the video frame's timestamp (in milliseconds). The input timestamps
// must be monotonically increasing.
absl::StatusOr<components::containers::proto::ClassificationResult>
ClassifyForVideo(mediapipe::Image image, int64 timestamp_ms,
std::optional<mediapipe::NormalizedRect>
image_processing_options = std::nullopt);
absl::StatusOr<ImageClassifierResult> ClassifyForVideo(
mediapipe::Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options =
std::nullopt);
// Sends live image data to image classification, and the results will be
// available via the "result_callback" provided in the ImageClassifierOptions.
//
// The optional 'image_processing_options' parameter can be used to specify:
// - the rotation to apply to the image before performing classification, by
// setting its 'rotation' field in radians (e.g. 'M_PI / 2' for a 90°
// anti-clockwise rotation).
// setting its 'rotation_degrees' field.
// and/or
// - the region-of-interest on which to perform classification, by setting its
// 'x_center', 'y_center', 'width' and 'height' fields. If none of these is
// set, they will automatically be set to cover the full image.
// 'region_of_interest' field. If not specified, the full image is used.
// If both are specified, the crop around the region-of-interest is extracted
// first, then the specified rotation is applied to the crop.
//
@@ -175,20 +172,17 @@ class ImageClassifier : tasks::vision::core::BaseVisionTaskApi {
// sent to the object detector. The input timestamps must be monotonically
// increasing.
//
// The "result_callback" prvoides
// - The classification results as a ClassificationResult object.
// The "result_callback" provides:
// - The classification results as an ImageClassifierResult object.
// - The const reference to the corresponding input image that the image
// classifier runs on. Note that the const reference to the image will no
// longer be valid when the callback returns. To access the image data
// outside of the callback, callers need to make a copy of the image.
// - The input timestamp in milliseconds.
absl::Status ClassifyAsync(mediapipe::Image image, int64 timestamp_ms,
std::optional<mediapipe::NormalizedRect>
std::optional<core::ImageProcessingOptions>
image_processing_options = std::nullopt);
// TODO: add Classify() variants taking a region of interest as
// additional argument.
// Shuts down the ImageClassifier when all works are done.
absl::Status Close() { return runner_->Close(); }
};
mediapipe/tasks/cc/vision/image_classifier/image_classifier_graph.cc
+15 -5
View File
@@ -48,6 +48,7 @@ using ::mediapipe::tasks::components::containers::proto::ClassificationResult;
constexpr float kDefaultScoreThreshold = std::numeric_limits<float>::lowest();
constexpr char kClassificationResultTag[] = "CLASSIFICATION_RESULT";
constexpr char kClassificationsTag[] = "CLASSIFICATIONS";
constexpr char kImageTag[] = "IMAGE";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kTensorsTag[] = "TENSORS";
@@ -56,6 +57,7 @@ constexpr char kTensorsTag[] = "TENSORS";
// subgraph.
struct ImageClassifierOutputStreams {
Source<ClassificationResult> classification_result;
Source<ClassificationResult> classifications;
Source<Image> image;
};
@@ -71,17 +73,19 @@ struct ImageClassifierOutputStreams {
// Describes region of image to perform classification on.
// @Optional: rect covering the whole image is used if not specified.
// Outputs:
// CLASSIFICATION_RESULT - ClassificationResult
// The aggregated classification result object has two dimensions:
// (classification head, classification category)
// CLASSIFICATIONS - ClassificationResult @Optional
// The classification results aggregated by classifier head.
// IMAGE - Image
// The image that object detection runs on.
// TODO: remove this output once Java API migration is over.
// CLASSIFICATION_RESULT - (DEPRECATED) ClassificationResult @Optional
// The aggregated classification result.
//
// Example:
// node {
// calculator: "mediapipe.tasks.vision.image_classifier.ImageClassifierGraph"
// input_stream: "IMAGE:image_in"
// output_stream: "CLASSIFICATION_RESULT:classification_result_out"
// output_stream: "CLASSIFICATIONS:classifications_out"
// output_stream: "IMAGE:image_out"
// options {
// [mediapipe.tasks.vision.image_classifier.proto.ImageClassifierGraphOptions.ext]
@@ -115,6 +119,8 @@ class ImageClassifierGraph : public core::ModelTaskGraph {
graph[Input<NormalizedRect>::Optional(kNormRectTag)], graph));
output_streams.classification_result >>
graph[Output<ClassificationResult>(kClassificationResultTag)];
output_streams.classifications >>
graph[Output<ClassificationResult>(kClassificationsTag)];
output_streams.image >> graph[Output<Image>(kImageTag)];
return graph.GetConfig();
}
@@ -138,8 +144,10 @@ class ImageClassifierGraph : public core::ModelTaskGraph {
// stream.
auto& preprocessing =
graph.AddNode("mediapipe.tasks.components.ImagePreprocessingSubgraph");
bool use_gpu = components::DetermineImagePreprocessingGpuBackend(
task_options.base_options().acceleration());
MP_RETURN_IF_ERROR(ConfigureImagePreprocessing(
model_resources,
model_resources, use_gpu,
&preprocessing
.GetOptions<tasks::components::ImagePreprocessingOptions>()));
image_in >> preprocessing.In(kImageTag);
@@ -168,6 +176,8 @@ class ImageClassifierGraph : public core::ModelTaskGraph {
return ImageClassifierOutputStreams{
/*classification_result=*/postprocessing[Output<ClassificationResult>(
kClassificationResultTag)],
/*classifications=*/
postprocessing[Output<ClassificationResult>(kClassificationsTag)],
/*image=*/preprocessing[Output<Image>(kImageTag)]};
}
};
mediapipe/tasks/cc/vision/image_classifier/image_classifier_test.cc
+204 -247
View File
@@ -27,14 +27,15 @@ limitations under the License.
#include "absl/strings/str_format.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/tasks/cc/common.h"
#include "mediapipe/tasks/cc/components/containers/proto/category.pb.h"
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
#include "mediapipe/tasks/cc/components/containers/category.h"
#include "mediapipe/tasks/cc/components/containers/classification_result.h"
#include "mediapipe/tasks/cc/components/containers/rect.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
#include "mediapipe/tasks/cc/vision/utils/image_utils.h"
#include "tensorflow/lite/core/api/op_resolver.h"
@@ -49,9 +50,10 @@ namespace image_classifier {
namespace {
using ::mediapipe::file::JoinPath;
using ::mediapipe::tasks::components::containers::proto::ClassificationEntry;
using ::mediapipe::tasks::components::containers::proto::ClassificationResult;
using ::mediapipe::tasks::components::containers::proto::Classifications;
using ::mediapipe::tasks::components::containers::Category;
using ::mediapipe::tasks::components::containers::Classifications;
using ::mediapipe::tasks::components::containers::Rect;
using ::mediapipe::tasks::vision::core::ImageProcessingOptions;
using ::testing::HasSubstr;
using ::testing::Optional;
@@ -62,83 +64,56 @@ constexpr char kMobileNetQuantizedWithMetadata[] =
constexpr char kMobileNetQuantizedWithDummyScoreCalibration[] =
"mobilenet_v1_0.25_224_quant_with_dummy_score_calibration.tflite";
// Checks that the two provided `ClassificationResult` are equal, with a
// Checks that the two provided `ImageClassifierResult` are equal, with a
// tolerancy on floating-point score to account for numerical instabilities.
void ExpectApproximatelyEqual(const ClassificationResult& actual,
const ClassificationResult& expected) {
void ExpectApproximatelyEqual(const ImageClassifierResult& actual,
const ImageClassifierResult& expected) {
const float kPrecision = 1e-6;
ASSERT_EQ(actual.classifications_size(), expected.classifications_size());
for (int i = 0; i < actual.classifications_size(); ++i) {
const Classifications& a = actual.classifications(i);
const Classifications& b = expected.classifications(i);
EXPECT_EQ(a.head_index(), b.head_index());
EXPECT_EQ(a.head_name(), b.head_name());
EXPECT_EQ(a.entries_size(), b.entries_size());
for (int j = 0; j < a.entries_size(); ++j) {
const ClassificationEntry& x = a.entries(j);
const ClassificationEntry& y = b.entries(j);
EXPECT_EQ(x.timestamp_ms(), y.timestamp_ms());
EXPECT_EQ(x.categories_size(), y.categories_size());
for (int k = 0; k < x.categories_size(); ++k) {
EXPECT_EQ(x.categories(k).index(), y.categories(k).index());
EXPECT_EQ(x.categories(k).category_name(),
y.categories(k).category_name());
EXPECT_EQ(x.categories(k).display_name(),
y.categories(k).display_name());
EXPECT_NEAR(x.categories(k).score(), y.categories(k).score(),
kPrecision);
}
ASSERT_EQ(actual.classifications.size(), expected.classifications.size());
for (int i = 0; i < actual.classifications.size(); ++i) {
const Classifications& a = actual.classifications[i];
const Classifications& b = expected.classifications[i];
EXPECT_EQ(a.head_index, b.head_index);
EXPECT_EQ(a.head_name, b.head_name);
EXPECT_EQ(a.categories.size(), b.categories.size());
for (int j = 0; j < a.categories.size(); ++j) {
const Category& x = a.categories[j];
const Category& y = b.categories[j];
EXPECT_EQ(x.index, y.index);
EXPECT_NEAR(x.score, y.score, kPrecision);
EXPECT_EQ(x.category_name, y.category_name);
EXPECT_EQ(x.display_name, y.display_name);
}
}
}
// Generates expected results for "burger.jpg" using kMobileNetFloatWithMetadata
// with max_results set to 3.
ClassificationResult GenerateBurgerResults(int64 timestamp) {
return ParseTextProtoOrDie<ClassificationResult>(
absl::StrFormat(R"pb(classifications {
entries {
categories {
index: 934
score: 0.7939592
category_name: "cheeseburger"
}
categories {
index: 932
score: 0.027392805
category_name: "bagel"
}
categories {
index: 925
score: 0.019340655
category_name: "guacamole"
}
timestamp_ms: %d
}
head_index: 0
head_name: "probability"
})pb",
timestamp));
ImageClassifierResult GenerateBurgerResults() {
ImageClassifierResult result;
result.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/934, /*score=*/0.793959200,
/*category_name=*/"cheeseburger"},
{/*index=*/932, /*score=*/0.027392805, /*category_name=*/"bagel"},
{/*index=*/925, /*score=*/0.019340655,
/*category_name=*/"guacamole"}},
/*head_index=*/0,
/*head_name=*/"probability"});
return result;
}
// Generates expected results for "multi_objects.jpg" using
// kMobileNetFloatWithMetadata with max_results set to 1 and the right bounding
// box set around the soccer ball.
ClassificationResult GenerateSoccerBallResults(int64 timestamp) {
return ParseTextProtoOrDie<ClassificationResult>(
absl::StrFormat(R"pb(classifications {
entries {
categories {
index: 806
score: 0.996527493
category_name: "soccer ball"
}
timestamp_ms: %d
}
head_index: 0
head_name: "probability"
})pb",
timestamp));
ImageClassifierResult GenerateSoccerBallResults() {
ImageClassifierResult result;
result.classifications.emplace_back(
Classifications{/*categories=*/{{/*index=*/806, /*score=*/0.996527493,
/*category_name=*/"soccer ball"}},
/*head_index=*/0,
/*head_name=*/"probability"});
return result;
}
// A custom OpResolver only containing the Ops required by the test model.
@@ -257,7 +232,7 @@ TEST_F(CreateTest, FailsWithIllegalCallbackInImageOrVideoMode) {
options->base_options.model_asset_path =
JoinPath("./", kTestDataDirectory, kMobileNetQuantizedWithMetadata);
options->running_mode = running_mode;
options->result_callback = [](absl::StatusOr<ClassificationResult>,
options->result_callback = [](absl::StatusOr<ImageClassifierResult>,
const Image& image, int64 timestamp_ms) {};
auto image_classifier = ImageClassifier::Create(std::move(options));
@@ -333,7 +308,7 @@ TEST_F(ImageModeTest, SucceedsWithFloatModel) {
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(image));
ExpectApproximatelyEqual(results, GenerateBurgerResults(0));
ExpectApproximatelyEqual(results, GenerateBurgerResults());
}
TEST_F(ImageModeTest, SucceedsWithQuantizedModel) {
@@ -352,19 +327,13 @@ TEST_F(ImageModeTest, SucceedsWithQuantizedModel) {
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(image));
ExpectApproximatelyEqual(results, ParseTextProtoOrDie<ClassificationResult>(
R"pb(classifications {
entries {
categories {
index: 934
score: 0.97265625
category_name: "cheeseburger"
}
timestamp_ms: 0
}
head_index: 0
head_name: "probability"
})pb"));
ImageClassifierResult expected;
expected.classifications.emplace_back(
Classifications{/*categories=*/{{/*index=*/934, /*score=*/0.97265625,
/*category_name=*/"cheeseburger"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);
}
TEST_F(ImageModeTest, SucceedsWithMaxResultsOption) {
@@ -380,19 +349,13 @@ TEST_F(ImageModeTest, SucceedsWithMaxResultsOption) {
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(image));
ExpectApproximatelyEqual(results, ParseTextProtoOrDie<ClassificationResult>(
R"pb(classifications {
entries {
categories {
index: 934
score: 0.7939592
category_name: "cheeseburger"
}
timestamp_ms: 0
}
head_index: 0
head_name: "probability"
})pb"));
ImageClassifierResult expected;
expected.classifications.emplace_back(
Classifications{/*categories=*/{{/*index=*/934, /*score=*/0.7939592,
/*category_name=*/"cheeseburger"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);
}
TEST_F(ImageModeTest, SucceedsWithScoreThresholdOption) {
@@ -408,24 +371,15 @@ TEST_F(ImageModeTest, SucceedsWithScoreThresholdOption) {
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(image));
ExpectApproximatelyEqual(results, ParseTextProtoOrDie<ClassificationResult>(
R"pb(classifications {
entries {
categories {
index: 934
score: 0.7939592
category_name: "cheeseburger"
}
categories {
index: 932
score: 0.027392805
category_name: "bagel"
}
timestamp_ms: 0
}
head_index: 0
head_name: "probability"
})pb"));
ImageClassifierResult expected;
expected.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/934, /*score=*/0.7939592,
/*category_name=*/"cheeseburger"},
{/*index=*/932, /*score=*/0.027392805, /*category_name=*/"bagel"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);
}
TEST_F(ImageModeTest, SucceedsWithAllowlistOption) {
@@ -442,29 +396,17 @@ TEST_F(ImageModeTest, SucceedsWithAllowlistOption) {
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(image));
ExpectApproximatelyEqual(results, ParseTextProtoOrDie<ClassificationResult>(
R"pb(classifications {
entries {
categories {
index: 934
score: 0.7939592
category_name: "cheeseburger"
}
categories {
index: 925
score: 0.019340655
category_name: "guacamole"
}
categories {
index: 963
score: 0.0063278517
category_name: "meat loaf"
}
timestamp_ms: 0
}
head_index: 0
head_name: "probability"
})pb"));
ImageClassifierResult expected;
expected.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/934, /*score=*/0.7939592,
/*category_name=*/"cheeseburger"},
{/*index=*/925, /*score=*/0.019340655, /*category_name=*/"guacamole"},
{/*index=*/963, /*score=*/0.0063278517,
/*category_name=*/"meat loaf"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);
}
TEST_F(ImageModeTest, SucceedsWithDenylistOption) {
@@ -481,29 +423,17 @@ TEST_F(ImageModeTest, SucceedsWithDenylistOption) {
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(image));
ExpectApproximatelyEqual(results, ParseTextProtoOrDie<ClassificationResult>(
R"pb(classifications {
entries {
categories {
index: 934
score: 0.7939592
category_name: "cheeseburger"
}
categories {
index: 925
score: 0.019340655
category_name: "guacamole"
}
categories {
index: 963
score: 0.0063278517
category_name: "meat loaf"
}
timestamp_ms: 0
}
head_index: 0
head_name: "probability"
})pb"));
ImageClassifierResult expected;
expected.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/934, /*score=*/0.7939592,
/*category_name=*/"cheeseburger"},
{/*index=*/925, /*score=*/0.019340655, /*category_name=*/"guacamole"},
{/*index=*/963, /*score=*/0.0063278517,
/*category_name=*/"meat loaf"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);
}
TEST_F(ImageModeTest, SucceedsWithScoreCalibration) {
@@ -522,19 +452,13 @@ TEST_F(ImageModeTest, SucceedsWithScoreCalibration) {
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(image));
ExpectApproximatelyEqual(results, ParseTextProtoOrDie<ClassificationResult>(
R"pb(classifications {
entries {
categories {
index: 934
score: 0.725648628
category_name: "cheeseburger"
}
timestamp_ms: 0
}
head_index: 0
head_name: "probability"
})pb"));
ImageClassifierResult expected;
expected.classifications.emplace_back(
Classifications{/*categories=*/{{/*index=*/934, /*score=*/0.725648628,
/*category_name=*/"cheeseburger"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);
}
TEST_F(ImageModeTest, SucceedsWithRegionOfInterest) {
@@ -547,17 +471,14 @@ TEST_F(ImageModeTest, SucceedsWithRegionOfInterest) {
options->classifier_options.max_results = 1;
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageClassifier> image_classifier,
ImageClassifier::Create(std::move(options)));
// Crop around the soccer ball.
NormalizedRect image_processing_options;
image_processing_options.set_x_center(0.532);
image_processing_options.set_y_center(0.521);
image_processing_options.set_width(0.164);
image_processing_options.set_height(0.427);
// Region-of-interest around the soccer ball.
Rect roi{/*left=*/0.45, /*top=*/0.3075, /*right=*/0.614, /*bottom=*/0.7345};
ImageProcessingOptions image_processing_options{roi, /*rotation_degrees=*/0};
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(
image, image_processing_options));
ExpectApproximatelyEqual(results, GenerateSoccerBallResults(0));
ExpectApproximatelyEqual(results, GenerateSoccerBallResults());
}
TEST_F(ImageModeTest, SucceedsWithRotation) {
@@ -572,8 +493,8 @@ TEST_F(ImageModeTest, SucceedsWithRotation) {
ImageClassifier::Create(std::move(options)));
// Specify a 90° anti-clockwise rotation.
NormalizedRect image_processing_options;
image_processing_options.set_rotation(M_PI / 2.0);
ImageProcessingOptions image_processing_options;
image_processing_options.rotation_degrees = -90;
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(
image, image_processing_options));
@@ -581,29 +502,17 @@ TEST_F(ImageModeTest, SucceedsWithRotation) {
// Results differ slightly from the non-rotated image, but that's expected
// as models are very sensitive to the slightest numerical differences
// introduced by the rotation and JPG encoding.
ExpectApproximatelyEqual(results, ParseTextProtoOrDie<ClassificationResult>(
R"pb(classifications {
entries {
categories {
index: 934
score: 0.6371766
category_name: "cheeseburger"
}
categories {
index: 963
score: 0.049443405
category_name: "meat loaf"
}
categories {
index: 925
score: 0.047918003
category_name: "guacamole"
}
timestamp_ms: 0
}
head_index: 0
head_name: "probability"
})pb"));
ImageClassifierResult expected;
expected.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/934, /*score=*/0.6371766,
/*category_name=*/"cheeseburger"},
{/*index=*/963, /*score=*/0.049443405, /*category_name=*/"meat loaf"},
{/*index=*/925, /*score=*/0.047918003,
/*category_name=*/"guacamole"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);
}
TEST_F(ImageModeTest, SucceedsWithRegionOfInterestAndRotation) {
@@ -616,31 +525,84 @@ TEST_F(ImageModeTest, SucceedsWithRegionOfInterestAndRotation) {
options->classifier_options.max_results = 1;
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageClassifier> image_classifier,
ImageClassifier::Create(std::move(options)));
// Crop around the chair, with 90° anti-clockwise rotation.
NormalizedRect image_processing_options;
image_processing_options.set_x_center(0.2821);
image_processing_options.set_y_center(0.2406);
image_processing_options.set_width(0.5642);
image_processing_options.set_height(0.1286);
image_processing_options.set_rotation(M_PI / 2.0);
// Region-of-interest around the chair, with 90° anti-clockwise rotation.
Rect roi{/*left=*/0.006, /*top=*/0.1763, /*right=*/0.5702, /*bottom=*/0.3049};
ImageProcessingOptions image_processing_options{roi,
/*rotation_degrees=*/-90};
MP_ASSERT_OK_AND_ASSIGN(auto results, image_classifier->Classify(
image, image_processing_options));
ExpectApproximatelyEqual(results,
ParseTextProtoOrDie<ClassificationResult>(
R"pb(classifications {
entries {
categories {
index: 560
score: 0.6800408
category_name: "folding chair"
}
timestamp_ms: 0
}
head_index: 0
head_name: "probability"
})pb"));
ImageClassifierResult expected;
expected.classifications.emplace_back(
Classifications{/*categories=*/{{/*index=*/560, /*score=*/0.6522213,
/*category_name=*/"folding chair"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);
}
// Testing all these once with ImageClassifier.
TEST_F(ImageModeTest, FailsWithInvalidImageProcessingOptions) {
MP_ASSERT_OK_AND_ASSIGN(Image image,
DecodeImageFromFile(JoinPath("./", kTestDataDirectory,
"multi_objects.jpg")));
auto options = std::make_unique<ImageClassifierOptions>();
options->base_options.model_asset_path =
JoinPath("./", kTestDataDirectory, kMobileNetFloatWithMetadata);
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageClassifier> image_classifier,
ImageClassifier::Create(std::move(options)));
// Invalid: left > right.
Rect roi{/*left=*/0.9, /*top=*/0, /*right=*/0.1, /*bottom=*/1};
ImageProcessingOptions image_processing_options{roi,
/*rotation_degrees=*/0};
auto results = image_classifier->Classify(image, image_processing_options);
EXPECT_EQ(results.status().code(), absl::StatusCode::kInvalidArgument);
EXPECT_THAT(results.status().message(),
HasSubstr("Expected Rect with left < right and top < bottom"));
EXPECT_THAT(
results.status().GetPayload(kMediaPipeTasksPayload),
Optional(absl::Cord(absl::StrCat(
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError))));
// Invalid: top > bottom.
roi = {/*left=*/0, /*top=*/0.9, /*right=*/1, /*bottom=*/0.1};
image_processing_options = {roi,
/*rotation_degrees=*/0};
results = image_classifier->Classify(image, image_processing_options);
EXPECT_EQ(results.status().code(), absl::StatusCode::kInvalidArgument);
EXPECT_THAT(results.status().message(),
HasSubstr("Expected Rect with left < right and top < bottom"));
EXPECT_THAT(
results.status().GetPayload(kMediaPipeTasksPayload),
Optional(absl::Cord(absl::StrCat(
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError))));
// Invalid: coordinates out of [0,1] range.
roi = {/*left=*/-0.1, /*top=*/0, /*right=*/1, /*bottom=*/1};
image_processing_options = {roi,
/*rotation_degrees=*/0};
results = image_classifier->Classify(image, image_processing_options);
EXPECT_EQ(results.status().code(), absl::StatusCode::kInvalidArgument);
EXPECT_THAT(results.status().message(),
HasSubstr("Expected Rect values to be in [0,1]"));
EXPECT_THAT(
results.status().GetPayload(kMediaPipeTasksPayload),
Optional(absl::Cord(absl::StrCat(
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError))));
// Invalid: rotation not a multiple of 90°.
image_processing_options = {/*region_of_interest=*/std::nullopt,
/*rotation_degrees=*/1};
results = image_classifier->Classify(image, image_processing_options);
EXPECT_EQ(results.status().code(), absl::StatusCode::kInvalidArgument);
EXPECT_THAT(results.status().message(),
HasSubstr("Expected rotation to be a multiple of 90°"));
EXPECT_THAT(
results.status().GetPayload(kMediaPipeTasksPayload),
Optional(absl::Cord(absl::StrCat(
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError))));
}
class VideoModeTest : public tflite_shims::testing::Test {};
@@ -714,7 +676,7 @@ TEST_F(VideoModeTest, Succeeds) {
for (int i = 0; i < iterations; ++i) {
MP_ASSERT_OK_AND_ASSIGN(auto results,
image_classifier->ClassifyForVideo(image, i));
ExpectApproximatelyEqual(results, GenerateBurgerResults(i));
ExpectApproximatelyEqual(results, GenerateBurgerResults());
}
MP_ASSERT_OK(image_classifier->Close());
}
@@ -732,17 +694,15 @@ TEST_F(VideoModeTest, SucceedsWithRegionOfInterest) {
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageClassifier> image_classifier,
ImageClassifier::Create(std::move(options)));
// Crop around the soccer ball.
NormalizedRect image_processing_options;
image_processing_options.set_x_center(0.532);
image_processing_options.set_y_center(0.521);
image_processing_options.set_width(0.164);
image_processing_options.set_height(0.427);
// Region-of-interest around the soccer ball.
Rect roi{/*left=*/0.45, /*top=*/0.3075, /*right=*/0.614, /*bottom=*/0.7345};
ImageProcessingOptions image_processing_options{roi, /*rotation_degrees=*/0};
for (int i = 0; i < iterations; ++i) {
MP_ASSERT_OK_AND_ASSIGN(
auto results,
image_classifier->ClassifyForVideo(image, i, image_processing_options));
ExpectApproximatelyEqual(results, GenerateSoccerBallResults(i));
ExpectApproximatelyEqual(results, GenerateSoccerBallResults());
}
MP_ASSERT_OK(image_classifier->Close());
}
@@ -757,7 +717,7 @@ TEST_F(LiveStreamModeTest, FailsWithCallingWrongMethod) {
options->base_options.model_asset_path =
JoinPath("./", kTestDataDirectory, kMobileNetFloatWithMetadata);
options->running_mode = core::RunningMode::LIVE_STREAM;
options->result_callback = [](absl::StatusOr<ClassificationResult>,
options->result_callback = [](absl::StatusOr<ImageClassifierResult>,
const Image& image, int64 timestamp_ms) {};
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageClassifier> image_classifier,
ImageClassifier::Create(std::move(options)));
@@ -788,7 +748,7 @@ TEST_F(LiveStreamModeTest, FailsWithOutOfOrderInputTimestamps) {
options->base_options.model_asset_path =
JoinPath("./", kTestDataDirectory, kMobileNetFloatWithMetadata);
options->running_mode = core::RunningMode::LIVE_STREAM;
options->result_callback = [](absl::StatusOr<ClassificationResult>,
options->result_callback = [](absl::StatusOr<ImageClassifierResult>,
const Image& image, int64 timestamp_ms) {};
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageClassifier> image_classifier,
ImageClassifier::Create(std::move(options)));
@@ -806,7 +766,7 @@ TEST_F(LiveStreamModeTest, FailsWithOutOfOrderInputTimestamps) {
}
struct LiveStreamModeResults {
ClassificationResult classification_result;
ImageClassifierResult classification_result;
std::pair<int, int> image_size;
int64 timestamp_ms;
};
@@ -823,7 +783,7 @@ TEST_F(LiveStreamModeTest, Succeeds) {
options->running_mode = core::RunningMode::LIVE_STREAM;
options->classifier_options.max_results = 3;
options->result_callback =
[&results](absl::StatusOr<ClassificationResult> classification_result,
[&results](absl::StatusOr<ImageClassifierResult> classification_result,
const Image& image, int64 timestamp_ms) {
MP_ASSERT_OK(classification_result.status());
results.push_back(
@@ -850,7 +810,7 @@ TEST_F(LiveStreamModeTest, Succeeds) {
EXPECT_EQ(result.image_size.first, image.width());
EXPECT_EQ(result.image_size.second, image.height());
ExpectApproximatelyEqual(result.classification_result,
GenerateBurgerResults(timestamp_ms));
GenerateBurgerResults());
}
}
@@ -866,7 +826,7 @@ TEST_F(LiveStreamModeTest, SucceedsWithRegionOfInterest) {
options->running_mode = core::RunningMode::LIVE_STREAM;
options->classifier_options.max_results = 1;
options->result_callback =
[&results](absl::StatusOr<ClassificationResult> classification_result,
[&results](absl::StatusOr<ImageClassifierResult> classification_result,
const Image& image, int64 timestamp_ms) {
MP_ASSERT_OK(classification_result.status());
results.push_back(
@@ -877,11 +837,8 @@ TEST_F(LiveStreamModeTest, SucceedsWithRegionOfInterest) {
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageClassifier> image_classifier,
ImageClassifier::Create(std::move(options)));
// Crop around the soccer ball.
NormalizedRect image_processing_options;
image_processing_options.set_x_center(0.532);
image_processing_options.set_y_center(0.521);
image_processing_options.set_width(0.164);
image_processing_options.set_height(0.427);
Rect roi{/*left=*/0.45, /*top=*/0.3075, /*right=*/0.614, /*bottom=*/0.7345};
ImageProcessingOptions image_processing_options{roi, /*rotation_degrees=*/0};
for (int i = 0; i < iterations; ++i) {
MP_ASSERT_OK(
@@ -900,7 +857,7 @@ TEST_F(LiveStreamModeTest, SucceedsWithRegionOfInterest) {
EXPECT_EQ(result.image_size.first, image.width());
EXPECT_EQ(result.image_size.second, image.height());
ExpectApproximatelyEqual(result.classification_result,
GenerateSoccerBallResults(timestamp_ms));
GenerateSoccerBallResults());
}
}
mediapipe/tasks/cc/vision/image_embedder/BUILD
+1
View File
@@ -58,6 +58,7 @@ cc_library(
"//mediapipe/tasks/cc/core:utils",
"//mediapipe/tasks/cc/core/proto:base_options_cc_proto",
"//mediapipe/tasks/cc/vision/core:base_vision_task_api",
"//mediapipe/tasks/cc/vision/core:image_processing_options",
"//mediapipe/tasks/cc/vision/core:running_mode",
"//mediapipe/tasks/cc/vision/core:vision_task_api_factory",
"//mediapipe/tasks/cc/vision/image_embedder/proto:image_embedder_graph_options_cc_proto",
mediapipe/tasks/cc/vision/image_embedder/image_embedder.cc
+14 -20
View File
@@ -29,6 +29,7 @@ limitations under the License.
#include "mediapipe/tasks/cc/core/proto/base_options.pb.h"
#include "mediapipe/tasks/cc/core/task_runner.h"
#include "mediapipe/tasks/cc/core/utils.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
#include "mediapipe/tasks/cc/vision/core/vision_task_api_factory.h"
#include "mediapipe/tasks/cc/vision/image_embedder/proto/image_embedder_graph_options.pb.h"
@@ -58,16 +59,6 @@ using ::mediapipe::tasks::core::PacketMap;
using ::mediapipe::tasks::vision::image_embedder::proto::
ImageEmbedderGraphOptions;
// Builds a NormalizedRect covering the entire image.
NormalizedRect BuildFullImageNormRect() {
NormalizedRect norm_rect;
norm_rect.set_x_center(0.5);
norm_rect.set_y_center(0.5);
norm_rect.set_width(1);
norm_rect.set_height(1);
return norm_rect;
}
// Creates a MediaPipe graph config that contains a single node of type
// "mediapipe.tasks.vision.image_embedder.ImageEmbedderGraph". If the task is
// running in the live stream mode, a "FlowLimiterCalculator" will be added to
@@ -148,15 +139,16 @@ absl::StatusOr<std::unique_ptr<ImageEmbedder>> ImageEmbedder::Create(
}
absl::StatusOr<EmbeddingResult> ImageEmbedder::Embed(
Image image, std::optional<NormalizedRect> roi) {
Image image,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"GPU input images are currently not supported.",
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
NormalizedRect norm_rect =
roi.has_value() ? roi.value() : BuildFullImageNormRect();
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@@ -167,15 +159,16 @@ absl::StatusOr<EmbeddingResult> ImageEmbedder::Embed(
}
absl::StatusOr<EmbeddingResult> ImageEmbedder::EmbedForVideo(
Image image, int64 timestamp_ms, std::optional<NormalizedRect> roi) {
Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"GPU input images are currently not supported.",
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
NormalizedRect norm_rect =
roi.has_value() ? roi.value() : BuildFullImageNormRect();
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@@ -188,16 +181,17 @@ absl::StatusOr<EmbeddingResult> ImageEmbedder::EmbedForVideo(
return output_packets[kEmbeddingResultStreamName].Get<EmbeddingResult>();
}
absl::Status ImageEmbedder::EmbedAsync(Image image, int64 timestamp_ms,
std::optional<NormalizedRect> roi) {
absl::Status ImageEmbedder::EmbedAsync(
Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"GPU input images are currently not supported.",
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
NormalizedRect norm_rect =
roi.has_value() ? roi.value() : BuildFullImageNormRect();
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))
mediapipe/tasks/cc/vision/image_embedder/image_embedder.h
+41 -15
View File
@@ -21,11 +21,11 @@ limitations under the License.
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/tasks/cc/components/containers/proto/embeddings.pb.h"
#include "mediapipe/tasks/cc/components/embedder_options.h"
#include "mediapipe/tasks/cc/core/base_options.h"
#include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
namespace mediapipe {
@@ -88,9 +88,17 @@ class ImageEmbedder : core::BaseVisionTaskApi {
static absl::StatusOr<std::unique_ptr<ImageEmbedder>> Create(
std::unique_ptr<ImageEmbedderOptions> options);
// Performs embedding extraction on the provided single image. Extraction
// is performed on the region of interest specified by the `roi` argument if
// provided, or on the entire image otherwise.
// Performs embedding extraction on the provided single image.
//
// The optional 'image_processing_options' parameter can be used to specify:
// - the rotation to apply to the image before performing embedding
// extraction, by setting its 'rotation_degrees' field.
// and/or
// - the region-of-interest on which to perform embedding extraction, by
// setting its 'region_of_interest' field. If not specified, the full image
// is used.
// If both are specified, the crop around the region-of-interest is extracted
// first, then the specified rotation is applied to the crop.
//
// Only use this method when the ImageEmbedder is created with the image
// running mode.
@@ -98,11 +106,20 @@ class ImageEmbedder : core::BaseVisionTaskApi {
// The image can be of any size with format RGB or RGBA.
absl::StatusOr<components::containers::proto::EmbeddingResult> Embed(
mediapipe::Image image,
std::optional<mediapipe::NormalizedRect> roi = std::nullopt);
std::optional<core::ImageProcessingOptions> image_processing_options =
std::nullopt);
// Performs embedding extraction on the provided video frame. Extraction
// is performed on the region of interested specified by the `roi` argument if
// provided, or on the entire image otherwise.
// Performs embedding extraction on the provided video frame.
//
// The optional 'image_processing_options' parameter can be used to specify:
// - the rotation to apply to the image before performing embedding
// extraction, by setting its 'rotation_degrees' field.
// and/or
// - the region-of-interest on which to perform embedding extraction, by
// setting its 'region_of_interest' field. If not specified, the full image
// is used.
// If both are specified, the crop around the region-of-interest is extracted
// first, then the specified rotation is applied to the crop.
//
// Only use this method when the ImageEmbedder is created with the video
// running mode.
@@ -112,12 +129,21 @@ class ImageEmbedder : core::BaseVisionTaskApi {
// must be monotonically increasing.
absl::StatusOr<components::containers::proto::EmbeddingResult> EmbedForVideo(
mediapipe::Image image, int64 timestamp_ms,
std::optional<mediapipe::NormalizedRect> roi = std::nullopt);
std::optional<core::ImageProcessingOptions> image_processing_options =
std::nullopt);
// Sends live image data to embedder, and the results will be available via
// the "result_callback" provided in the ImageEmbedderOptions. Embedding
// extraction is performed on the region of interested specified by the `roi`
// argument if provided, or on the entire image otherwise.
// the "result_callback" provided in the ImageEmbedderOptions.
//
// The optional 'image_processing_options' parameter can be used to specify:
// - the rotation to apply to the image before performing embedding
// extraction, by setting its 'rotation_degrees' field.
// and/or
// - the region-of-interest on which to perform embedding extraction, by
// setting its 'region_of_interest' field. If not specified, the full image
// is used.
// If both are specified, the crop around the region-of-interest is extracted
// first, then the specified rotation is applied to the crop.
//
// Only use this method when the ImageEmbedder is created with the live
// stream running mode.
@@ -135,9 +161,9 @@ class ImageEmbedder : core::BaseVisionTaskApi {
// longer be valid when the callback returns. To access the image data
// outside of the callback, callers need to make a copy of the image.
// - The input timestamp in milliseconds.
absl::Status EmbedAsync(
mediapipe::Image image, int64 timestamp_ms,
std::optional<mediapipe::NormalizedRect> roi = std::nullopt);
absl::Status EmbedAsync(mediapipe::Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions>
image_processing_options = std::nullopt);
// Shuts down the ImageEmbedder when all works are done.
absl::Status Close() { return runner_->Close(); }
mediapipe/tasks/cc/vision/image_embedder/image_embedder_graph.cc
+3 -1
View File
@@ -134,8 +134,10 @@ class ImageEmbedderGraph : public core::ModelTaskGraph {
// stream.
auto& preprocessing =
graph.AddNode("mediapipe.tasks.components.ImagePreprocessingSubgraph");
bool use_gpu = components::DetermineImagePreprocessingGpuBackend(
task_options.base_options().acceleration());
MP_RETURN_IF_ERROR(ConfigureImagePreprocessing(
model_resources,
model_resources, use_gpu,
&preprocessing
.GetOptions<tasks::components::ImagePreprocessingOptions>()));
image_in >> preprocessing.In(kImageTag);
mediapipe/tasks/cc/vision/image_embedder/image_embedder_test.cc
+79 -9
View File
@@ -23,7 +23,6 @@ limitations under the License.
#include "absl/status/statusor.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
@@ -42,7 +41,9 @@ namespace image_embedder {
namespace {
using ::mediapipe::file::JoinPath;
using ::mediapipe::tasks::components::containers::Rect;
using ::mediapipe::tasks::components::containers::proto::EmbeddingResult;
using ::mediapipe::tasks::vision::core::ImageProcessingOptions;
using ::testing::HasSubstr;
using ::testing::Optional;
@@ -326,16 +327,14 @@ TEST_F(ImageModeTest, SucceedsWithRegionOfInterest) {
MP_ASSERT_OK_AND_ASSIGN(
Image crop, DecodeImageFromFile(
JoinPath("./", kTestDataDirectory, "burger_crop.jpg")));
// Bounding box in "burger.jpg" corresponding to "burger_crop.jpg".
NormalizedRect roi;
roi.set_x_center(200.0 / 480);
roi.set_y_center(0.5);
roi.set_width(400.0 / 480);
roi.set_height(1.0f);
// Region-of-interest in "burger.jpg" corresponding to "burger_crop.jpg".
Rect roi{/*left=*/0, /*top=*/0, /*right=*/0.833333, /*bottom=*/1};
ImageProcessingOptions image_processing_options{roi, /*rotation_degrees=*/0};
// Extract both embeddings.
MP_ASSERT_OK_AND_ASSIGN(const EmbeddingResult& image_result,
image_embedder->Embed(image, roi));
MP_ASSERT_OK_AND_ASSIGN(
const EmbeddingResult& image_result,
image_embedder->Embed(image, image_processing_options));
MP_ASSERT_OK_AND_ASSIGN(const EmbeddingResult& crop_result,
image_embedder->Embed(crop));
@@ -351,6 +350,77 @@ TEST_F(ImageModeTest, SucceedsWithRegionOfInterest) {
EXPECT_LE(abs(similarity - expected_similarity), kSimilarityTolerancy);
}
TEST_F(ImageModeTest, SucceedsWithRotation) {
auto options = std::make_unique<ImageEmbedderOptions>();
options->base_options.model_asset_path =
JoinPath("./", kTestDataDirectory, kMobileNetV3Embedder);
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageEmbedder> image_embedder,
ImageEmbedder::Create(std::move(options)));
// Load images: one is a rotated version of the other.
MP_ASSERT_OK_AND_ASSIGN(
Image image,
DecodeImageFromFile(JoinPath("./", kTestDataDirectory, "burger.jpg")));
MP_ASSERT_OK_AND_ASSIGN(Image rotated,
DecodeImageFromFile(JoinPath("./", kTestDataDirectory,
"burger_rotated.jpg")));
ImageProcessingOptions image_processing_options;
image_processing_options.rotation_degrees = -90;
// Extract both embeddings.
MP_ASSERT_OK_AND_ASSIGN(const EmbeddingResult& image_result,
image_embedder->Embed(image));
MP_ASSERT_OK_AND_ASSIGN(
const EmbeddingResult& rotated_result,
image_embedder->Embed(rotated, image_processing_options));
// Check results.
CheckMobileNetV3Result(image_result, false);
CheckMobileNetV3Result(rotated_result, false);
// CheckCosineSimilarity.
MP_ASSERT_OK_AND_ASSIGN(
double similarity,
ImageEmbedder::CosineSimilarity(image_result.embeddings(0).entries(0),
rotated_result.embeddings(0).entries(0)));
double expected_similarity = 0.572265;
EXPECT_LE(abs(similarity - expected_similarity), kSimilarityTolerancy);
}
TEST_F(ImageModeTest, SucceedsWithRegionOfInterestAndRotation) {
auto options = std::make_unique<ImageEmbedderOptions>();
options->base_options.model_asset_path =
JoinPath("./", kTestDataDirectory, kMobileNetV3Embedder);
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageEmbedder> image_embedder,
ImageEmbedder::Create(std::move(options)));
MP_ASSERT_OK_AND_ASSIGN(
Image crop, DecodeImageFromFile(
JoinPath("./", kTestDataDirectory, "burger_crop.jpg")));
MP_ASSERT_OK_AND_ASSIGN(Image rotated,
DecodeImageFromFile(JoinPath("./", kTestDataDirectory,
"burger_rotated.jpg")));
// Region-of-interest corresponding to burger_crop.jpg.
Rect roi{/*left=*/0, /*top=*/0, /*right=*/1, /*bottom=*/0.8333333};
ImageProcessingOptions image_processing_options{roi,
/*rotation_degrees=*/-90};
// Extract both embeddings.
MP_ASSERT_OK_AND_ASSIGN(const EmbeddingResult& crop_result,
image_embedder->Embed(crop));
MP_ASSERT_OK_AND_ASSIGN(
const EmbeddingResult& rotated_result,
image_embedder->Embed(rotated, image_processing_options));
// Check results.
CheckMobileNetV3Result(crop_result, false);
CheckMobileNetV3Result(rotated_result, false);
// CheckCosineSimilarity.
MP_ASSERT_OK_AND_ASSIGN(
double similarity,
ImageEmbedder::CosineSimilarity(crop_result.embeddings(0).entries(0),
rotated_result.embeddings(0).entries(0)));
double expected_similarity = 0.62838;
EXPECT_LE(abs(similarity - expected_similarity), kSimilarityTolerancy);
}
class VideoModeTest : public tflite_shims::testing::Test {};
TEST_F(VideoModeTest, FailsWithCallingWrongMethod) {
mediapipe/tasks/cc/vision/image_segmenter/BUILD
+3
View File
@@ -24,10 +24,12 @@ cc_library(
":image_segmenter_graph",
"//mediapipe/framework/api2:builder",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/tasks/cc/components/proto:segmenter_options_cc_proto",
"//mediapipe/tasks/cc/core:base_options",
"//mediapipe/tasks/cc/core:utils",
"//mediapipe/tasks/cc/vision/core:base_vision_task_api",
"//mediapipe/tasks/cc/vision/core:image_processing_options",
"//mediapipe/tasks/cc/vision/core:running_mode",
"//mediapipe/tasks/cc/vision/core:vision_task_api_factory",
"//mediapipe/tasks/cc/vision/image_segmenter/proto:image_segmenter_options_cc_proto",
@@ -48,6 +50,7 @@ cc_library(
"//mediapipe/framework/api2:builder",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:status",
"//mediapipe/tasks/cc:common",
"//mediapipe/tasks/cc/components:image_preprocessing",
mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.cc
+35 -7
View File
@@ -17,8 +17,10 @@ limitations under the License.
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/tasks/cc/components/proto/segmenter_options.pb.h"
#include "mediapipe/tasks/cc/core/utils.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
#include "mediapipe/tasks/cc/vision/core/vision_task_api_factory.h"
@@ -32,6 +34,8 @@ constexpr char kGroupedSegmentationTag[] = "GROUPED_SEGMENTATION";
constexpr char kImageInStreamName[] = "image_in";
constexpr char kImageOutStreamName[] = "image_out";
constexpr char kImageTag[] = "IMAGE";
constexpr char kNormRectStreamName[] = "norm_rect_in";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kSubgraphTypeName[] =
"mediapipe.tasks.vision.ImageSegmenterGraph";
constexpr int kMicroSecondsPerMilliSecond = 1000;
@@ -51,15 +55,18 @@ CalculatorGraphConfig CreateGraphConfig(
auto& task_subgraph = graph.AddNode(kSubgraphTypeName);
task_subgraph.GetOptions<ImageSegmenterOptionsProto>().Swap(options.get());
graph.In(kImageTag).SetName(kImageInStreamName);
graph.In(kNormRectTag).SetName(kNormRectStreamName);
task_subgraph.Out(kGroupedSegmentationTag).SetName(kSegmentationStreamName) >>
graph.Out(kGroupedSegmentationTag);
task_subgraph.Out(kImageTag).SetName(kImageOutStreamName) >>
graph.Out(kImageTag);
if (enable_flow_limiting) {
return tasks::core::AddFlowLimiterCalculator(
graph, task_subgraph, {kImageTag}, kGroupedSegmentationTag);
return tasks::core::AddFlowLimiterCalculator(graph, task_subgraph,
{kImageTag, kNormRectTag},
kGroupedSegmentationTag);
}
graph.In(kImageTag) >> task_subgraph.In(kImageTag);
graph.In(kNormRectTag) >> task_subgraph.In(kNormRectTag);
return graph.GetConfig();
}
@@ -139,47 +146,68 @@ absl::StatusOr<std::unique_ptr<ImageSegmenter>> ImageSegmenter::Create(
}
absl::StatusOr<std::vector<Image>> ImageSegmenter::Segment(
mediapipe::Image image) {
mediapipe::Image image,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData({{kImageInStreamName,
mediapipe::MakePacket<Image>(std::move(image))}}));
ProcessImageData(
{{kImageInStreamName, mediapipe::MakePacket<Image>(std::move(image))},
{kNormRectStreamName,
MakePacket<NormalizedRect>(std::move(norm_rect))}}));
return output_packets[kSegmentationStreamName].Get<std::vector<Image>>();
}
absl::StatusOr<std::vector<Image>> ImageSegmenter::SegmentForVideo(
mediapipe::Image image, int64 timestamp_ms) {
mediapipe::Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))
.At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))},
{kNormRectStreamName,
MakePacket<NormalizedRect>(std::move(norm_rect))
.At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}}));
return output_packets[kSegmentationStreamName].Get<std::vector<Image>>();
}
absl::Status ImageSegmenter::SegmentAsync(Image image, int64 timestamp_ms) {
absl::Status ImageSegmenter::SegmentAsync(
Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))
.At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))},
{kNormRectStreamName,
MakePacket<NormalizedRect>(std::move(norm_rect))
.At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}});
}
mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.h
+29 -5
View File
@@ -25,6 +25,7 @@ limitations under the License.
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/tasks/cc/core/base_options.h"
#include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/image_segmenter/proto/image_segmenter_options.pb.h"
#include "tensorflow/lite/kernels/register.h"
@@ -116,14 +117,21 @@ class ImageSegmenter : tasks::vision::core::BaseVisionTaskApi {
// running mode.
//
// The image can be of any size with format RGB or RGBA.
// TODO: Describes how the input image will be preprocessed
// after the yuv support is implemented.
//
// The optional 'image_processing_options' parameter can be used to specify
// the rotation to apply to the image before performing segmentation, by
// setting its 'rotation_degrees' field. Note that specifying a
// region-of-interest using the 'region_of_interest' field is NOT supported
// and will result in an invalid argument error being returned.
//
// If the output_type is CATEGORY_MASK, the returned vector of images is
// per-category segmented image mask.
// If the output_type is CONFIDENCE_MASK, the returned vector of images
// contains only one confidence image mask.
absl::StatusOr<std::vector<mediapipe::Image>> Segment(mediapipe::Image image);
absl::StatusOr<std::vector<mediapipe::Image>> Segment(
mediapipe::Image image,
std::optional<core::ImageProcessingOptions> image_processing_options =
std::nullopt);
// Performs image segmentation on the provided video frame.
// Only use this method when the ImageSegmenter is created with the video
@@ -133,12 +141,20 @@ class ImageSegmenter : tasks::vision::core::BaseVisionTaskApi {
// provide the video frame's timestamp (in milliseconds). The input timestamps
// must be monotonically increasing.
//
// The optional 'image_processing_options' parameter can be used to specify
// the rotation to apply to the image before performing segmentation, by
// setting its 'rotation_degrees' field. Note that specifying a
// region-of-interest using the 'region_of_interest' field is NOT supported
// and will result in an invalid argument error being returned.
//
// If the output_type is CATEGORY_MASK, the returned vector of images is
// per-category segmented image mask.
// If the output_type is CONFIDENCE_MASK, the returned vector of images
// contains only one confidence image mask.
absl::StatusOr<std::vector<mediapipe::Image>> SegmentForVideo(
mediapipe::Image image, int64 timestamp_ms);
mediapipe::Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options =
std::nullopt);
// Sends live image data to perform image segmentation, and the results will
// be available via the "result_callback" provided in the
@@ -150,6 +166,12 @@ class ImageSegmenter : tasks::vision::core::BaseVisionTaskApi {
// sent to the image segmenter. The input timestamps must be monotonically
// increasing.
//
// The optional 'image_processing_options' parameter can be used to specify
// the rotation to apply to the image before performing segmentation, by
// setting its 'rotation_degrees' field. Note that specifying a
// region-of-interest using the 'region_of_interest' field is NOT supported
// and will result in an invalid argument error being returned.
//
// The "result_callback" prvoides
// - A vector of segmented image masks.
// If the output_type is CATEGORY_MASK, the returned vector of images is
@@ -161,7 +183,9 @@ class ImageSegmenter : tasks::vision::core::BaseVisionTaskApi {
// no longer be valid when the callback returns. To access the image data
// outside of the callback, callers need to make a copy of the image.
// - The input timestamp in milliseconds.
absl::Status SegmentAsync(mediapipe::Image image, int64 timestamp_ms);
absl::Status SegmentAsync(mediapipe::Image image, int64 timestamp_ms,
std::optional<core::ImageProcessingOptions>
image_processing_options = std::nullopt);
// Shuts down the ImageSegmenter when all works are done.
absl::Status Close() { return runner_->Close(); }
mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_graph.cc
+17 -6
View File
@@ -23,6 +23,7 @@ limitations under the License.
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/tasks/cc/common.h"
#include "mediapipe/tasks/cc/components/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
@@ -62,6 +63,7 @@ using LabelItems = mediapipe::proto_ns::Map<int64, ::mediapipe::LabelMapItem>;
constexpr char kSegmentationTag[] = "SEGMENTATION";
constexpr char kGroupedSegmentationTag[] = "GROUPED_SEGMENTATION";
constexpr char kImageTag[] = "IMAGE";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kTensorsTag[] = "TENSORS";
constexpr char kOutputSizeTag[] = "OUTPUT_SIZE";
@@ -159,6 +161,10 @@ absl::StatusOr<const Tensor*> GetOutputTensor(
// Inputs:
// IMAGE - Image
// Image to perform segmentation on.
// NORM_RECT - NormalizedRect @Optional
// Describes image rotation and region of image to perform detection
// on.
// @Optional: rect covering the whole image is used if not specified.
//
// Outputs:
// SEGMENTATION - mediapipe::Image @Multiple
@@ -196,10 +202,12 @@ class ImageSegmenterGraph : public core::ModelTaskGraph {
ASSIGN_OR_RETURN(const auto* model_resources,
CreateModelResources<ImageSegmenterOptions>(sc));
Graph graph;
ASSIGN_OR_RETURN(auto output_streams,
BuildSegmentationTask(
sc->Options<ImageSegmenterOptions>(), *model_resources,
graph[Input<Image>(kImageTag)], graph));
ASSIGN_OR_RETURN(
auto output_streams,
BuildSegmentationTask(
sc->Options<ImageSegmenterOptions>(), *model_resources,
graph[Input<Image>(kImageTag)],
graph[Input<NormalizedRect>::Optional(kNormRectTag)], graph));
auto& merge_images_to_vector =
graph.AddNode("MergeImagesToVectorCalculator");
@@ -228,18 +236,21 @@ class ImageSegmenterGraph : public core::ModelTaskGraph {
absl::StatusOr<ImageSegmenterOutputs> BuildSegmentationTask(
const ImageSegmenterOptions& task_options,
const core::ModelResources& model_resources, Source<Image> image_in,
Graph& graph) {
Source<NormalizedRect> norm_rect_in, Graph& graph) {
MP_RETURN_IF_ERROR(SanityCheckOptions(task_options));
// Adds preprocessing calculators and connects them to the graph input image
// stream.
auto& preprocessing =
graph.AddNode("mediapipe.tasks.components.ImagePreprocessingSubgraph");
bool use_gpu = components::DetermineImagePreprocessingGpuBackend(
task_options.base_options().acceleration());
MP_RETURN_IF_ERROR(ConfigureImagePreprocessing(
model_resources,
model_resources, use_gpu,
&preprocessing
.GetOptions<tasks::components::ImagePreprocessingOptions>()));
image_in >> preprocessing.In(kImageTag);
norm_rect_in >> preprocessing.In(kNormRectTag);
// Adds inference subgraph and connects its input stream to the output
// tensors produced by the ImageToTensorCalculator.
mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_test.cc
+59 -1
View File
@@ -29,8 +29,10 @@ limitations under the License.
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/tasks/cc/components/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/tasks/cc/components/containers/rect.h"
#include "mediapipe/tasks/cc/core/proto/base_options.pb.h"
#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/image_segmenter/proto/image_segmenter_options.pb.h"
#include "mediapipe/tasks/cc/vision/utils/image_utils.h"
#include "tensorflow/lite/core/shims/cc/shims_test_util.h"
@@ -44,6 +46,8 @@ namespace {
using ::mediapipe::Image;
using ::mediapipe::file::JoinPath;
using ::mediapipe::tasks::components::containers::Rect;
using ::mediapipe::tasks::vision::core::ImageProcessingOptions;
using ::testing::HasSubstr;
using ::testing::Optional;
@@ -237,7 +241,6 @@ TEST_F(ImageModeTest, SucceedsWithConfidenceMask) {
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageSegmenter> segmenter,
ImageSegmenter::Create(std::move(options)));
MP_ASSERT_OK_AND_ASSIGN(auto results, segmenter->Segment(image));
MP_ASSERT_OK_AND_ASSIGN(auto confidence_masks, segmenter->Segment(image));
EXPECT_EQ(confidence_masks.size(), 21);
@@ -253,6 +256,61 @@ TEST_F(ImageModeTest, SucceedsWithConfidenceMask) {
SimilarToFloatMask(expected_mask_float, kGoldenMaskSimilarity));
}
TEST_F(ImageModeTest, SucceedsWithRotation) {
MP_ASSERT_OK_AND_ASSIGN(
Image image, DecodeImageFromFile(
JoinPath("./", kTestDataDirectory, "cat_rotated.jpg")));
auto options = std::make_unique<ImageSegmenterOptions>();
options->base_options.model_asset_path =
JoinPath("./", kTestDataDirectory, kDeeplabV3WithMetadata);
options->output_type = ImageSegmenterOptions::OutputType::CONFIDENCE_MASK;
options->activation = ImageSegmenterOptions::Activation::SOFTMAX;
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageSegmenter> segmenter,
ImageSegmenter::Create(std::move(options)));
ImageProcessingOptions image_processing_options;
image_processing_options.rotation_degrees = -90;
MP_ASSERT_OK_AND_ASSIGN(auto confidence_masks, segmenter->Segment(image));
EXPECT_EQ(confidence_masks.size(), 21);
cv::Mat expected_mask =
cv::imread(JoinPath("./", kTestDataDirectory, "cat_rotated_mask.jpg"),
cv::IMREAD_GRAYSCALE);
cv::Mat expected_mask_float;
expected_mask.convertTo(expected_mask_float, CV_32FC1, 1 / 255.f);
// Cat category index 8.
cv::Mat cat_mask = mediapipe::formats::MatView(
confidence_masks[8].GetImageFrameSharedPtr().get());
EXPECT_THAT(cat_mask,
SimilarToFloatMask(expected_mask_float, kGoldenMaskSimilarity));
}
TEST_F(ImageModeTest, FailsWithRegionOfInterest) {
MP_ASSERT_OK_AND_ASSIGN(
Image image,
DecodeImageFromFile(JoinPath("./", kTestDataDirectory, "cat.jpg")));
auto options = std::make_unique<ImageSegmenterOptions>();
options->base_options.model_asset_path =
JoinPath("./", kTestDataDirectory, kDeeplabV3WithMetadata);
options->output_type = ImageSegmenterOptions::OutputType::CONFIDENCE_MASK;
options->activation = ImageSegmenterOptions::Activation::SOFTMAX;
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageSegmenter> segmenter,
ImageSegmenter::Create(std::move(options)));
Rect roi{/*left=*/0.1, /*top=*/0, /*right=*/0.9, /*bottom=*/1};
ImageProcessingOptions image_processing_options{roi, /*rotation_degrees=*/0};
auto results = segmenter->Segment(image, image_processing_options);
EXPECT_EQ(results.status().code(), absl::StatusCode::kInvalidArgument);
EXPECT_THAT(results.status().message(),
HasSubstr("This task doesn't support region-of-interest"));
EXPECT_THAT(
results.status().GetPayload(kMediaPipeTasksPayload),
Optional(absl::Cord(absl::StrCat(
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError))));
}
TEST_F(ImageModeTest, SucceedsSelfie128x128Segmentation) {
Image image =
GetSRGBImage(JoinPath("./", kTestDataDirectory, "mozart_square.jpg"));
mediapipe/tasks/cc/vision/object_detector/BUILD
+1
View File
@@ -75,6 +75,7 @@ cc_library(
"//mediapipe/tasks/cc/core/proto:base_options_cc_proto",
"//mediapipe/tasks/cc/core/proto:inference_subgraph_cc_proto",
"//mediapipe/tasks/cc/vision/core:base_vision_task_api",
"//mediapipe/tasks/cc/vision/core:image_processing_options",
"//mediapipe/tasks/cc/vision/core:running_mode",
"//mediapipe/tasks/cc/vision/core:vision_task_api_factory",
"//mediapipe/tasks/cc/vision/object_detector/proto:object_detector_options_cc_proto",
mediapipe/tasks/cc/vision/object_detector/object_detector.cc
+13 -34
View File
@@ -34,6 +34,7 @@ limitations under the License.
#include "mediapipe/tasks/cc/core/proto/base_options.pb.h"
#include "mediapipe/tasks/cc/core/proto/inference_subgraph.pb.h"
#include "mediapipe/tasks/cc/core/utils.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
#include "mediapipe/tasks/cc/vision/core/vision_task_api_factory.h"
#include "mediapipe/tasks/cc/vision/object_detector/proto/object_detector_options.pb.h"
@@ -58,31 +59,6 @@ constexpr int kMicroSecondsPerMilliSecond = 1000;
using ObjectDetectorOptionsProto =
object_detector::proto::ObjectDetectorOptions;
// Returns a NormalizedRect filling the whole image. If input is present, its
// rotation is set in the returned NormalizedRect and a check is performed to
// make sure no region-of-interest was provided. Otherwise, rotation is set to
// 0.
absl::StatusOr<NormalizedRect> FillNormalizedRect(
std::optional<NormalizedRect> normalized_rect) {
NormalizedRect result;
if (normalized_rect.has_value()) {
result = *normalized_rect;
}
bool has_coordinates = result.has_x_center() || result.has_y_center() ||
result.has_width() || result.has_height();
if (has_coordinates) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
"ObjectDetector does not support region-of-interest.",
MediaPipeTasksStatus::kInvalidArgumentError);
}
result.set_x_center(0.5);
result.set_y_center(0.5);
result.set_width(1);
result.set_height(1);
return result;
}
// Creates a MediaPipe graph config that contains a subgraph node of
// "mediapipe.tasks.vision.ObjectDetectorGraph". If the task is running in the
// live stream mode, a "FlowLimiterCalculator" will be added to limit the
@@ -170,15 +146,16 @@ absl::StatusOr<std::unique_ptr<ObjectDetector>> ObjectDetector::Create(
absl::StatusOr<std::vector<Detection>> ObjectDetector::Detect(
mediapipe::Image image,
std::optional<mediapipe::NormalizedRect> image_processing_options) {
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
FillNormalizedRect(image_processing_options));
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@@ -189,15 +166,16 @@ absl::StatusOr<std::vector<Detection>> ObjectDetector::Detect(
absl::StatusOr<std::vector<Detection>> ObjectDetector::DetectForVideo(
mediapipe::Image image, int64 timestamp_ms,
std::optional<mediapipe::NormalizedRect> image_processing_options) {
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
FillNormalizedRect(image_processing_options));
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@@ -212,15 +190,16 @@ absl::StatusOr<std::vector<Detection>> ObjectDetector::DetectForVideo(
absl::Status ObjectDetector::DetectAsync(
Image image, int64 timestamp_ms,
std::optional<mediapipe::NormalizedRect> image_processing_options) {
std::optional<core::ImageProcessingOptions> image_processing_options) {
if (image.UsesGpu()) {
return CreateStatusWithPayload(
absl::StatusCode::kInvalidArgument,
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
FillNormalizedRect(image_processing_options));
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))
mediapipe/tasks/cc/vision/object_detector/object_detector.h
+13 -16
View File
@@ -27,9 +27,9 @@ limitations under the License.
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/tasks/cc/core/base_options.h"
#include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
namespace mediapipe {
@@ -154,10 +154,9 @@ class ObjectDetector : tasks::vision::core::BaseVisionTaskApi {
// after the yuv support is implemented.
//
// The optional 'image_processing_options' parameter can be used to specify
// the rotation to apply to the image before performing classification, by
// setting its 'rotation' field in radians (e.g. 'M_PI / 2' for a 90°
// anti-clockwise rotation). Note that specifying a region-of-interest using
// the 'x_center', 'y_center', 'width' and 'height' fields is NOT supported
// the rotation to apply to the image before performing detection, by
// setting its 'rotation_degrees' field. Note that specifying a
// region-of-interest using the 'region_of_interest' field is NOT supported
// and will result in an invalid argument error being returned.
//
// For CPU images, the returned bounding boxes are expressed in the
@@ -168,7 +167,7 @@ class ObjectDetector : tasks::vision::core::BaseVisionTaskApi {
// images after enabling the gpu support in MediaPipe Tasks.
absl::StatusOr<std::vector<mediapipe::Detection>> Detect(
mediapipe::Image image,
std::optional<mediapipe::NormalizedRect> image_processing_options =
std::optional<core::ImageProcessingOptions> image_processing_options =
std::nullopt);
// Performs object detection on the provided video frame.
@@ -180,10 +179,9 @@ class ObjectDetector : tasks::vision::core::BaseVisionTaskApi {
// must be monotonically increasing.
//
// The optional 'image_processing_options' parameter can be used to specify
// the rotation to apply to the image before performing classification, by
// setting its 'rotation' field in radians (e.g. 'M_PI / 2' for a 90°
// anti-clockwise rotation). Note that specifying a region-of-interest using
// the 'x_center', 'y_center', 'width' and 'height' fields is NOT supported
// the rotation to apply to the image before performing detection, by
// setting its 'rotation_degrees' field. Note that specifying a
// region-of-interest using the 'region_of_interest' field is NOT supported
// and will result in an invalid argument error being returned.
//
// For CPU images, the returned bounding boxes are expressed in the
@@ -192,7 +190,7 @@ class ObjectDetector : tasks::vision::core::BaseVisionTaskApi {
// underlying image data.
absl::StatusOr<std::vector<mediapipe::Detection>> DetectForVideo(
mediapipe::Image image, int64 timestamp_ms,
std::optional<mediapipe::NormalizedRect> image_processing_options =
std::optional<core::ImageProcessingOptions> image_processing_options =
std::nullopt);
// Sends live image data to perform object detection, and the results will be
@@ -206,10 +204,9 @@ class ObjectDetector : tasks::vision::core::BaseVisionTaskApi {
// increasing.
//
// The optional 'image_processing_options' parameter can be used to specify
// the rotation to apply to the image before performing classification, by
// setting its 'rotation' field in radians (e.g. 'M_PI / 2' for a 90°
// anti-clockwise rotation). Note that specifying a region-of-interest using
// the 'x_center', 'y_center', 'width' and 'height' fields is NOT supported
// the rotation to apply to the image before performing detection, by
// setting its 'rotation_degrees' field. Note that specifying a
// region-of-interest using the 'region_of_interest' field is NOT supported
// and will result in an invalid argument error being returned.
//
// The "result_callback" provides
@@ -223,7 +220,7 @@ class ObjectDetector : tasks::vision::core::BaseVisionTaskApi {
// outside of the callback, callers need to make a copy of the image.
// - The input timestamp in milliseconds.
absl::Status DetectAsync(mediapipe::Image image, int64 timestamp_ms,
std::optional<mediapipe::NormalizedRect>
std::optional<core::ImageProcessingOptions>
image_processing_options = std::nullopt);
// Shuts down the ObjectDetector when all works are done.
mediapipe/tasks/cc/vision/object_detector/object_detector_graph.cc
+3 -1
View File
@@ -563,8 +563,10 @@ class ObjectDetectorGraph : public core::ModelTaskGraph {
// stream.
auto& preprocessing =
graph.AddNode("mediapipe.tasks.components.ImagePreprocessingSubgraph");
bool use_gpu = components::DetermineImagePreprocessingGpuBackend(
task_options.base_options().acceleration());
MP_RETURN_IF_ERROR(ConfigureImagePreprocessing(
model_resources,
model_resources, use_gpu,
&preprocessing
.GetOptions<tasks::components::ImagePreprocessingOptions>()));
image_in >> preprocessing.In(kImageTag);
mediapipe/tasks/cc/vision/object_detector/object_detector_test.cc
+13 -9
View File
@@ -31,11 +31,12 @@ limitations under the License.
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/location_data.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/tasks/cc/components/containers/rect.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
#include "mediapipe/tasks/cc/vision/utils/image_utils.h"
#include "tensorflow/lite/c/common.h"
@@ -64,6 +65,8 @@ namespace vision {
namespace {
using ::mediapipe::file::JoinPath;
using ::mediapipe::tasks::components::containers::Rect;
using ::mediapipe::tasks::vision::core::ImageProcessingOptions;
using ::testing::HasSubstr;
using ::testing::Optional;
@@ -532,8 +535,8 @@ TEST_F(ImageModeTest, SucceedsWithRotation) {
JoinPath("./", kTestDataDirectory, kMobileSsdWithMetadata);
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ObjectDetector> object_detector,
ObjectDetector::Create(std::move(options)));
NormalizedRect image_processing_options;
image_processing_options.set_rotation(M_PI / 2.0);
ImageProcessingOptions image_processing_options;
image_processing_options.rotation_degrees = -90;
MP_ASSERT_OK_AND_ASSIGN(
auto results, object_detector->Detect(image, image_processing_options));
MP_ASSERT_OK(object_detector->Close());
@@ -557,16 +560,17 @@ TEST_F(ImageModeTest, FailsWithRegionOfInterest) {
JoinPath("./", kTestDataDirectory, kMobileSsdWithMetadata);
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ObjectDetector> object_detector,
ObjectDetector::Create(std::move(options)));
NormalizedRect image_processing_options;
image_processing_options.set_x_center(0.5);
image_processing_options.set_y_center(0.5);
image_processing_options.set_width(1.0);
image_processing_options.set_height(1.0);
Rect roi{/*left=*/0.1, /*top=*/0, /*right=*/0.9, /*bottom=*/1};
ImageProcessingOptions image_processing_options{roi, /*rotation_degrees=*/0};
auto results = object_detector->Detect(image, image_processing_options);
EXPECT_EQ(results.status().code(), absl::StatusCode::kInvalidArgument);
EXPECT_THAT(results.status().message(),
HasSubstr("ObjectDetector does not support region-of-interest"));
HasSubstr("This task doesn't support region-of-interest"));
EXPECT_THAT(
results.status().GetPayload(kMediaPipeTasksPayload),
Optional(absl::Cord(absl::StrCat(
MediaPipeTasksStatus::kImageProcessingInvalidArgumentError))));
}
class VideoModeTest : public tflite_shims::testing::Test {};
mediapipe/tasks/examples/android/objectdetector/src/main/BUILD
+1
View File
@@ -31,6 +31,7 @@ android_binary(
multidex = "native",
resource_files = ["//mediapipe/tasks/examples/android:resource_files"],
deps = [
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
"//mediapipe/java/com/google/mediapipe/framework/image",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:detection",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
mediapipe/tasks/examples/android/objectdetector/src/main/java/com/google/mediapipe/tasks/examples/objectdetector/MainActivity.java
+24 -21
View File
@@ -16,7 +16,6 @@ package com.google.mediapipe.tasks.examples.objectdetector;
import android.content.Intent;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.media.MediaMetadataRetriever;
import android.os.Bundle;
import android.provider.MediaStore;
@@ -29,9 +28,11 @@ import androidx.activity.result.ActivityResultLauncher;
import androidx.activity.result.contract.ActivityResultContracts;
import androidx.exifinterface.media.ExifInterface;
// ContentResolver dependency
import com.google.mediapipe.framework.MediaPipeException;
import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.Image;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
import com.google.mediapipe.tasks.vision.core.RunningMode;
import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetectionResult;
import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetector;
@@ -82,6 +83,7 @@ public class MainActivity extends AppCompatActivity {
if (resultIntent != null) {
if (result.getResultCode() == RESULT_OK) {
Bitmap bitmap = null;
int rotation = 0;
try {
bitmap =
downscaleBitmap(
@@ -93,13 +95,16 @@ public class MainActivity extends AppCompatActivity {
try {
InputStream imageData =
this.getContentResolver().openInputStream(resultIntent.getData());
bitmap = rotateBitmap(bitmap, imageData);
} catch (IOException e) {
rotation = getImageRotation(imageData);
} catch (IOException | MediaPipeException e) {
Log.e(TAG, "Bitmap rotation error:" + e);
}
if (bitmap != null) {
Image image = new BitmapImageBuilder(bitmap).build();
ObjectDetectionResult detectionResult = objectDetector.detect(image);
MPImage image = new BitmapImageBuilder(bitmap).build();
ObjectDetectionResult detectionResult =
objectDetector.detect(
image,
ImageProcessingOptions.builder().setRotationDegrees(rotation).build());
imageView.setData(image, detectionResult);
runOnUiThread(() -> imageView.update());
}
@@ -144,7 +149,8 @@ public class MainActivity extends AppCompatActivity {
MediaMetadataRetriever.METADATA_KEY_VIDEO_FRAME_COUNT));
long frameIntervalMs = duration / numFrames;
for (int i = 0; i < numFrames; ++i) {
Image image = new BitmapImageBuilder(metaRetriever.getFrameAtIndex(i)).build();
MPImage image =
new BitmapImageBuilder(metaRetriever.getFrameAtIndex(i)).build();
ObjectDetectionResult detectionResult =
objectDetector.detectForVideo(image, frameIntervalMs * i);
// Currently only annotates the detection result on the first video frame and
@@ -209,28 +215,25 @@ public class MainActivity extends AppCompatActivity {
return Bitmap.createScaledBitmap(originalBitmap, width, height, false);
}
private Bitmap rotateBitmap(Bitmap inputBitmap, InputStream imageData) throws IOException {
private int getImageRotation(InputStream imageData) throws IOException, MediaPipeException {
int orientation =
new ExifInterface(imageData)
.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
if (orientation == ExifInterface.ORIENTATION_NORMAL) {
return inputBitmap;
}
Matrix matrix = new Matrix();
switch (orientation) {
case ExifInterface.ORIENTATION_NORMAL:
return 0;
case ExifInterface.ORIENTATION_ROTATE_90:
matrix.postRotate(90);
break;
return 90;
case ExifInterface.ORIENTATION_ROTATE_180:
matrix.postRotate(180);
break;
return 180;
case ExifInterface.ORIENTATION_ROTATE_270:
matrix.postRotate(270);
break;
return 270;
default:
matrix.postRotate(0);
// TODO: use getRotationDegrees() and isFlipped() instead of switch once flip
// is supported.
throw new MediaPipeException(
MediaPipeException.StatusCode.UNIMPLEMENTED.ordinal(),
"Flipped images are not supported yet.");
}
return Bitmap.createBitmap(
inputBitmap, 0, 0, inputBitmap.getWidth(), inputBitmap.getHeight(), matrix, true);
}
}
mediapipe/tasks/examples/android/objectdetector/src/main/java/com/google/mediapipe/tasks/examples/objectdetector/ObjectDetectionResultImageView.java
+4 -4
View File
@@ -22,7 +22,7 @@ import android.graphics.Matrix;
import android.graphics.Paint;
import androidx.appcompat.widget.AppCompatImageView;
import com.google.mediapipe.framework.image.BitmapExtractor;
import com.google.mediapipe.framework.image.Image;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.containers.Detection;
import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetectionResult;
@@ -40,12 +40,12 @@ public class ObjectDetectionResultImageView extends AppCompatImageView {
}
/**
* Sets an {@link Image} and an {@link ObjectDetectionResult} to render.
* Sets a {@link MPImage} and an {@link ObjectDetectionResult} to render.
*
* @param image an {@link Image} object for annotation.
* @param image a {@link MPImage} object for annotation.
* @param result an {@link ObjectDetectionResult} object that contains the detection result.
*/
public void setData(Image image, ObjectDetectionResult result) {
public void setData(MPImage image, ObjectDetectionResult result) {
if (image == null || result == null) {
return;
}
mediapipe/tasks/java/BUILD
+15 -1
View File
@@ -1 +1,15 @@
# dummy file for tap test to find the pattern
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
mediapipe/tasks/java/com/google/mediapipe/tasks/BUILD
+15
View File
@@ -0,0 +1,15 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java
+3 -3
View File
@@ -30,13 +30,13 @@ public abstract class Landmark {
return new AutoValue_Landmark(x, y, z, normalized);
}
// The x coordniates of the landmark.
// The x coordinates of the landmark.
public abstract float x();
// The y coordniates of the landmark.
// The y coordinates of the landmark.
public abstract float y();
// The z coordniates of the landmark.
// The z coordinates of the landmark.
public abstract float z();
// Whether this landmark is normalized with respect to the image size.
mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD
+12
View File
@@ -36,3 +36,15 @@ android_library(
"@maven//:com_google_guava_guava",
],
)
load("//mediapipe/tasks/java/com/google/mediapipe/tasks:mediapipe_tasks_aar.bzl", "mediapipe_tasks_core_aar")
mediapipe_tasks_core_aar(
name = "tasks_core",
srcs = glob(["*.java"]) + [
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:java_src",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors:java_src",
"//mediapipe/java/com/google/mediapipe/framework/image:java_src",
],
manifest = "AndroidManifest.xml",
)
mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl
+256
View File
@@ -0,0 +1,256 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Building MediaPipe Tasks AARs."""
load("//mediapipe/java/com/google/mediapipe:mediapipe_aar.bzl", "mediapipe_build_aar_with_jni", "mediapipe_java_proto_src_extractor", "mediapipe_java_proto_srcs")
load("@build_bazel_rules_android//android:rules.bzl", "android_library")
_CORE_TASKS_JAVA_PROTO_LITE_TARGETS = [
"//mediapipe/tasks/cc/components/containers/proto:category_java_proto_lite",
"//mediapipe/tasks/cc/components/containers/proto:classifications_java_proto_lite",
"//mediapipe/tasks/cc/components/containers/proto:embeddings_java_proto_lite",
"//mediapipe/tasks/cc/components/containers/proto:landmarks_detection_result_java_proto_lite",
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_java_proto_lite",
"//mediapipe/tasks/cc/core/proto:acceleration_java_proto_lite",
"//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite",
"//mediapipe/tasks/cc/core/proto:external_file_java_proto_lite",
]
_VISION_TASKS_JAVA_PROTO_LITE_TARGETS = [
"//mediapipe/tasks/cc/vision/object_detector/proto:object_detector_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/image_classifier/proto:image_classifier_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_embedder_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite",
]
_TEXT_TASKS_JAVA_PROTO_LITE_TARGETS = [
"//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_java_proto_lite",
]
def mediapipe_tasks_core_aar(name, srcs, manifest):
"""Builds medaipipe tasks core AAR.
Args:
name: The bazel target name.
srcs: MediaPipe Tasks' core layer source files.
manifest: The Android manifest.
"""
mediapipe_tasks_java_proto_srcs = []
for target in _CORE_TASKS_JAVA_PROTO_LITE_TARGETS:
mediapipe_tasks_java_proto_srcs.append(
_mediapipe_tasks_java_proto_src_extractor(target = target),
)
for target in _VISION_TASKS_JAVA_PROTO_LITE_TARGETS:
mediapipe_tasks_java_proto_srcs.append(
_mediapipe_tasks_java_proto_src_extractor(target = target),
)
for target in _TEXT_TASKS_JAVA_PROTO_LITE_TARGETS:
mediapipe_tasks_java_proto_srcs.append(
_mediapipe_tasks_java_proto_src_extractor(target = target),
)
mediapipe_tasks_java_proto_srcs.append(mediapipe_java_proto_src_extractor(
target = "//mediapipe/calculators/core:flow_limiter_calculator_java_proto_lite",
src_out = "com/google/mediapipe/calculator/proto/FlowLimiterCalculatorProto.java",
))
mediapipe_tasks_java_proto_srcs.append(mediapipe_java_proto_src_extractor(
target = "//mediapipe/calculators/tensor:inference_calculator_java_proto_lite",
src_out = "com/google/mediapipe/calculator/proto/InferenceCalculatorProto.java",
))
android_library(
name = name,
srcs = srcs + [
"//mediapipe/java/com/google/mediapipe/framework:java_src",
] + mediapipe_java_proto_srcs() +
mediapipe_tasks_java_proto_srcs,
javacopts = [
"-Xep:AndroidJdkLibsChecker:OFF",
],
manifest = manifest,
deps = [
"//mediapipe/calculators/core:flow_limiter_calculator_java_proto_lite",
"//mediapipe/calculators/tensor:inference_calculator_java_proto_lite",
"//mediapipe/framework:calculator_java_proto_lite",
"//mediapipe/framework:calculator_profile_java_proto_lite",
"//mediapipe/framework:calculator_options_java_proto_lite",
"//mediapipe/framework:mediapipe_options_java_proto_lite",
"//mediapipe/framework:packet_factory_java_proto_lite",
"//mediapipe/framework:packet_generator_java_proto_lite",
"//mediapipe/framework:status_handler_java_proto_lite",
"//mediapipe/framework:stream_handler_java_proto_lite",
"//mediapipe/framework/formats:classification_java_proto_lite",
"//mediapipe/framework/formats:detection_java_proto_lite",
"//mediapipe/framework/formats:landmark_java_proto_lite",
"//mediapipe/framework/formats:location_data_java_proto_lite",
"//mediapipe/framework/formats:rect_java_proto_lite",
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
"//mediapipe/java/com/google/mediapipe/framework/image",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni:model_resources_cache_jni",
"//third_party:androidx_annotation",
"//third_party:autovalue",
"@com_google_protobuf//:protobuf_javalite",
"@maven//:com_google_guava_guava",
"@maven//:com_google_flogger_flogger",
"@maven//:com_google_flogger_flogger_system_backend",
"@maven//:com_google_code_findbugs_jsr305",
] +
_CORE_TASKS_JAVA_PROTO_LITE_TARGETS +
_VISION_TASKS_JAVA_PROTO_LITE_TARGETS +
_TEXT_TASKS_JAVA_PROTO_LITE_TARGETS,
)
def mediapipe_tasks_vision_aar(name, srcs, native_library):
"""Builds medaipipe tasks vision AAR.
Args:
name: The bazel target name.
srcs: MediaPipe Vision Tasks' source files.
native_library: The native library that contains vision tasks' graph and calculators.
"""
native.genrule(
name = name + "tasks_manifest_generator",
outs = ["AndroidManifest.xml"],
cmd = """
cat > $(OUTS) <<EOF
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.google.mediapipe.tasks.vision">
<uses-sdk
android:minSdkVersion="24"
android:targetSdkVersion="30" />
</manifest>
EOF
""",
)
_mediapipe_tasks_aar(
name = name,
srcs = srcs,
manifest = "AndroidManifest.xml",
java_proto_lite_targets = _CORE_TASKS_JAVA_PROTO_LITE_TARGETS + _VISION_TASKS_JAVA_PROTO_LITE_TARGETS,
native_library = native_library,
)
def mediapipe_tasks_text_aar(name, srcs, native_library):
"""Builds medaipipe tasks text AAR.
Args:
name: The bazel target name.
srcs: MediaPipe Text Tasks' source files.
native_library: The native library that contains text tasks' graph and calculators.
"""
native.genrule(
name = name + "tasks_manifest_generator",
outs = ["AndroidManifest.xml"],
cmd = """
cat > $(OUTS) <<EOF
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.google.mediapipe.tasks.text">
<uses-sdk
android:minSdkVersion="24"
android:targetSdkVersion="30" />
</manifest>
EOF
""",
)
_mediapipe_tasks_aar(
name = name,
srcs = srcs,
manifest = "AndroidManifest.xml",
java_proto_lite_targets = _CORE_TASKS_JAVA_PROTO_LITE_TARGETS + _TEXT_TASKS_JAVA_PROTO_LITE_TARGETS,
native_library = native_library,
)
def _mediapipe_tasks_aar(name, srcs, manifest, java_proto_lite_targets, native_library):
"""Builds medaipipe tasks AAR."""
# When "--define EXCLUDE_OPENCV_SO_LIB=1" is set in the build command,
# the OpenCV so libraries will be excluded from the AAR package to
# save the package size.
native.config_setting(
name = "exclude_opencv_so_lib",
define_values = {
"EXCLUDE_OPENCV_SO_LIB": "1",
},
visibility = ["//visibility:public"],
)
native.cc_library(
name = name + "_jni_opencv_cc_lib",
srcs = select({
"//mediapipe:android_arm64": ["@android_opencv//:libopencv_java3_so_arm64-v8a"],
"//mediapipe:android_armeabi": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"],
"//mediapipe:android_arm": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"],
"//mediapipe:android_x86": ["@android_opencv//:libopencv_java3_so_x86"],
"//mediapipe:android_x86_64": ["@android_opencv//:libopencv_java3_so_x86_64"],
"//conditions:default": [],
}),
alwayslink = 1,
)
android_library(
name = name + "_android_lib",
srcs = srcs,
manifest = manifest,
proguard_specs = ["//mediapipe/java/com/google/mediapipe/framework:proguard.pgcfg"],
deps = java_proto_lite_targets + [native_library] + [
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
"//mediapipe/java/com/google/mediapipe/framework/image",
"//mediapipe/framework:calculator_options_java_proto_lite",
"//mediapipe/framework:calculator_java_proto_lite",
"//mediapipe/framework/formats:classification_java_proto_lite",
"//mediapipe/framework/formats:detection_java_proto_lite",
"//mediapipe/framework/formats:landmark_java_proto_lite",
"//mediapipe/framework/formats:location_data_java_proto_lite",
"//mediapipe/framework/formats:rect_java_proto_lite",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:detection",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:classification_entry",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:classifications",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors:classifieroptions",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
"//third_party:autovalue",
"@maven//:com_google_guava_guava",
] + select({
"//conditions:default": [":" + name + "_jni_opencv_cc_lib"],
"//mediapipe/framework/port:disable_opencv": [],
"exclude_opencv_so_lib": [],
}),
)
mediapipe_build_aar_with_jni(name, name + "_android_lib")
def _mediapipe_tasks_java_proto_src_extractor(target):
proto_path = "com/google/" + target.split(":")[0].replace("cc/", "").replace("//", "").replace("_", "") + "/"
proto_name = target.split(":")[-1].replace("_java_proto_lite", "").replace("_", " ").title().replace(" ", "") + "Proto.java"
return mediapipe_java_proto_src_extractor(
target = target,
src_out = proto_path + proto_name,
)
mediapipe/tasks/java/com/google/mediapipe/tasks/text/BUILD
+8
View File
@@ -61,3 +61,11 @@ android_library(
"@maven//:com_google_guava_guava",
],
)
load("//mediapipe/tasks/java/com/google/mediapipe/tasks:mediapipe_tasks_aar.bzl", "mediapipe_tasks_text_aar")
mediapipe_tasks_text_aar(
name = "tasks_text",
srcs = glob(["**/*.java"]),
native_library = ":libmediapipe_tasks_text_jni_lib",
)
mediapipe/tasks/java/com/google/mediapipe/tasks/text/textclassifier/TextClassificationResult.java
+2 -2
View File
@@ -15,11 +15,11 @@
package com.google.mediapipe.tasks.text.textclassifier;
import com.google.auto.value.AutoValue;
import com.google.mediapipe.tasks.components.container.proto.CategoryProto;
import com.google.mediapipe.tasks.components.container.proto.ClassificationsProto;
import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.containers.ClassificationEntry;
import com.google.mediapipe.tasks.components.containers.Classifications;
import com.google.mediapipe.tasks.components.containers.proto.CategoryProto;
import com.google.mediapipe.tasks.components.containers.proto.ClassificationsProto;
import com.google.mediapipe.tasks.core.TaskResult;
import java.util.ArrayList;
import java.util.Collections;
mediapipe/tasks/java/com/google/mediapipe/tasks/text/textclassifier/TextClassifier.java
+2 -3
View File
@@ -22,7 +22,7 @@ import com.google.mediapipe.framework.MediaPipeException;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.PacketGetter;
import com.google.mediapipe.framework.ProtoUtil;
import com.google.mediapipe.tasks.components.container.proto.ClassificationsProto;
import com.google.mediapipe.tasks.components.containers.proto.ClassificationsProto;
import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.core.OutputHandler;
@@ -31,7 +31,6 @@ import com.google.mediapipe.tasks.core.TaskOptions;
import com.google.mediapipe.tasks.core.TaskRunner;
import com.google.mediapipe.tasks.core.proto.BaseOptionsProto;
import com.google.mediapipe.tasks.text.textclassifier.proto.TextClassifierGraphOptionsProto;
import com.google.protobuf.InvalidProtocolBufferException;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
@@ -154,7 +153,7 @@ public final class TextClassifier implements AutoCloseable {
packets.get(CLASSIFICATION_RESULT_OUT_STREAM_INDEX),
ClassificationsProto.ClassificationResult.getDefaultInstance()),
packets.get(CLASSIFICATION_RESULT_OUT_STREAM_INDEX).getTimestamp());
} catch (InvalidProtocolBufferException e) {
} catch (IOException e) {
throw new MediaPipeException(
MediaPipeException.StatusCode.INTERNAL.ordinal(), e.getMessage());
}
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD
+11
View File
@@ -28,6 +28,7 @@ android_library(
"//mediapipe/java/com/google/mediapipe/framework:android_framework_no_mff",
"//mediapipe/java/com/google/mediapipe/framework/image",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
"//third_party:autovalue",
"@maven//:com_google_guava_guava",
],
)
@@ -128,6 +129,7 @@ android_library(
"//mediapipe/java/com/google/mediapipe/framework/image",
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_java_proto_lite",
"//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_java_proto_lite",
"//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_java_proto_lite",
@@ -135,8 +137,17 @@ android_library(
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors:classifieroptions",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
"//third_party:autovalue",
"@maven//:com_google_guava_guava",
],
)
load("//mediapipe/tasks/java/com/google/mediapipe/tasks:mediapipe_tasks_aar.bzl", "mediapipe_tasks_vision_aar")
mediapipe_tasks_vision_aar(
name = "tasks_vision",
srcs = glob(["**/*.java"]),
native_library = ":libmediapipe_tasks_vision_jni_lib",
)
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/core/BaseVisionTaskApi.java
+46 -143
View File
@@ -19,12 +19,11 @@ import com.google.mediapipe.formats.proto.RectProto.NormalizedRect;
import com.google.mediapipe.framework.MediaPipeException;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.ProtoUtil;
import com.google.mediapipe.framework.image.Image;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.core.TaskResult;
import com.google.mediapipe.tasks.core.TaskRunner;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
/** The base class of MediaPipe vision tasks. */
public class BaseVisionTaskApi implements AutoCloseable {
@@ -32,7 +31,7 @@ public class BaseVisionTaskApi implements AutoCloseable {
private final TaskRunner runner;
private final RunningMode runningMode;
private final String imageStreamName;
private final Optional<String> normRectStreamName;
private final String normRectStreamName;
static {
System.loadLibrary("mediapipe_tasks_vision_jni");
@@ -40,27 +39,13 @@ public class BaseVisionTaskApi implements AutoCloseable {
}
/**
* Constructor to initialize a {@link BaseVisionTaskApi} only taking images as input.
* Constructor to initialize a {@link BaseVisionTaskApi}.
*
* @param runner a {@link TaskRunner}.
* @param runningMode a mediapipe vision task {@link RunningMode}.
* @param imageStreamName the name of the input image stream.
*/
public BaseVisionTaskApi(TaskRunner runner, RunningMode runningMode, String imageStreamName) {
this.runner = runner;
this.runningMode = runningMode;
this.imageStreamName = imageStreamName;
this.normRectStreamName = Optional.empty();
}
/**
* Constructor to initialize a {@link BaseVisionTaskApi} taking images and normalized rects as
* input.
*
* @param runner a {@link TaskRunner}.
* @param runningMode a mediapipe vision task {@link RunningMode}.
* @param imageStreamName the name of the input image stream.
* @param normRectStreamName the name of the input normalized rect image stream.
* @param normRectStreamName the name of the input normalized rect image stream used to provide
* (mandatory) rotation and (optional) region-of-interest.
*/
public BaseVisionTaskApi(
TaskRunner runner,
@@ -70,61 +55,31 @@ public class BaseVisionTaskApi implements AutoCloseable {
this.runner = runner;
this.runningMode = runningMode;
this.imageStreamName = imageStreamName;
this.normRectStreamName = Optional.of(normRectStreamName);
this.normRectStreamName = normRectStreamName;
}
/**
* A synchronous method to process single image inputs. The call blocks the current thread until a
* failure status or a successful result is returned.
*
* @param image a MediaPipe {@link Image} object for processing.
* @throws MediaPipeException if the task is not in the image mode or requires a normalized rect
* input.
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference.
* @throws MediaPipeException if the task is not in the image mode.
*/
protected TaskResult processImageData(Image image) {
protected TaskResult processImageData(
MPImage image, ImageProcessingOptions imageProcessingOptions) {
if (runningMode != RunningMode.IMAGE) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task is not initialized with the image mode. Current running mode:"
+ runningMode.name());
}
if (normRectStreamName.isPresent()) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task expects a normalized rect as input.");
}
Map<String, Packet> inputPackets = new HashMap<>();
inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image));
return runner.process(inputPackets);
}
/**
* A synchronous method to process single image inputs. The call blocks the current thread until a
* failure status or a successful result is returned.
*
* @param image a MediaPipe {@link Image} object for processing.
* @param roi a {@link RectF} defining the region-of-interest to process in the image. Coordinates
* are expected to be specified as normalized values in [0,1].
* @throws MediaPipeException if the task is not in the image mode or doesn't require a normalized
* rect.
*/
protected TaskResult processImageData(Image image, RectF roi) {
if (runningMode != RunningMode.IMAGE) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task is not initialized with the image mode. Current running mode:"
+ runningMode.name());
}
if (!normRectStreamName.isPresent()) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task doesn't expect a normalized rect as input.");
}
Map<String, Packet> inputPackets = new HashMap<>();
inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image));
inputPackets.put(
normRectStreamName.get(),
runner.getPacketCreator().createProto(convertToNormalizedRect(roi)));
normRectStreamName,
runner.getPacketCreator().createProto(convertToNormalizedRect(imageProcessingOptions)));
return runner.process(inputPackets);
}
@@ -132,56 +87,25 @@ public class BaseVisionTaskApi implements AutoCloseable {
* A synchronous method to process continuous video frames. The call blocks the current thread
* until a failure status or a successful result is returned.
*
* @param image a MediaPipe {@link Image} object for processing.
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference.
* @param timestampMs the corresponding timestamp of the input image in milliseconds.
* @throws MediaPipeException if the task is not in the video mode or requires a normalized rect
* input.
* @throws MediaPipeException if the task is not in the video mode.
*/
protected TaskResult processVideoData(Image image, long timestampMs) {
protected TaskResult processVideoData(
MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
if (runningMode != RunningMode.VIDEO) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task is not initialized with the video mode. Current running mode:"
+ runningMode.name());
}
if (normRectStreamName.isPresent()) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task expects a normalized rect as input.");
}
Map<String, Packet> inputPackets = new HashMap<>();
inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image));
return runner.process(inputPackets, timestampMs * MICROSECONDS_PER_MILLISECOND);
}
/**
* A synchronous method to process continuous video frames. The call blocks the current thread
* until a failure status or a successful result is returned.
*
* @param image a MediaPipe {@link Image} object for processing.
* @param roi a {@link RectF} defining the region-of-interest to process in the image. Coordinates
* are expected to be specified as normalized values in [0,1].
* @param timestampMs the corresponding timestamp of the input image in milliseconds.
* @throws MediaPipeException if the task is not in the video mode or doesn't require a normalized
* rect.
*/
protected TaskResult processVideoData(Image image, RectF roi, long timestampMs) {
if (runningMode != RunningMode.VIDEO) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task is not initialized with the video mode. Current running mode:"
+ runningMode.name());
}
if (!normRectStreamName.isPresent()) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task doesn't expect a normalized rect as input.");
}
Map<String, Packet> inputPackets = new HashMap<>();
inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image));
inputPackets.put(
normRectStreamName.get(),
runner.getPacketCreator().createProto(convertToNormalizedRect(roi)));
normRectStreamName,
runner.getPacketCreator().createProto(convertToNormalizedRect(imageProcessingOptions)));
return runner.process(inputPackets, timestampMs * MICROSECONDS_PER_MILLISECOND);
}
@@ -189,56 +113,25 @@ public class BaseVisionTaskApi implements AutoCloseable {
* An asynchronous method to send live stream data to the {@link TaskRunner}. The results will be
* available in the user-defined result listener.
*
* @param image a MediaPipe {@link Image} object for processing.
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference.
* @param timestampMs the corresponding timestamp of the input image in milliseconds.
* @throws MediaPipeException if the task is not in the video mode or requires a normalized rect
* input.
* @throws MediaPipeException if the task is not in the stream mode.
*/
protected void sendLiveStreamData(Image image, long timestampMs) {
protected void sendLiveStreamData(
MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
if (runningMode != RunningMode.LIVE_STREAM) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task is not initialized with the live stream mode. Current running mode:"
+ runningMode.name());
}
if (normRectStreamName.isPresent()) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task expects a normalized rect as input.");
}
Map<String, Packet> inputPackets = new HashMap<>();
inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image));
runner.send(inputPackets, timestampMs * MICROSECONDS_PER_MILLISECOND);
}
/**
* An asynchronous method to send live stream data to the {@link TaskRunner}. The results will be
* available in the user-defined result listener.
*
* @param image a MediaPipe {@link Image} object for processing.
* @param roi a {@link RectF} defining the region-of-interest to process in the image. Coordinates
* are expected to be specified as normalized values in [0,1].
* @param timestampMs the corresponding timestamp of the input image in milliseconds.
* @throws MediaPipeException if the task is not in the video mode or doesn't require a normalized
* rect.
*/
protected void sendLiveStreamData(Image image, RectF roi, long timestampMs) {
if (runningMode != RunningMode.LIVE_STREAM) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task is not initialized with the live stream mode. Current running mode:"
+ runningMode.name());
}
if (!normRectStreamName.isPresent()) {
throw new MediaPipeException(
MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
"Task doesn't expect a normalized rect as input.");
}
Map<String, Packet> inputPackets = new HashMap<>();
inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image));
inputPackets.put(
normRectStreamName.get(),
runner.getPacketCreator().createProto(convertToNormalizedRect(roi)));
normRectStreamName,
runner.getPacketCreator().createProto(convertToNormalizedRect(imageProcessingOptions)));
runner.send(inputPackets, timestampMs * MICROSECONDS_PER_MILLISECOND);
}
@@ -248,13 +141,23 @@ public class BaseVisionTaskApi implements AutoCloseable {
runner.close();
}
/** Converts a {@link RectF} object into a {@link NormalizedRect} protobuf message. */
private static NormalizedRect convertToNormalizedRect(RectF rect) {
/**
* Converts an {@link ImageProcessingOptions} instance into a {@link NormalizedRect} protobuf
* message.
*/
private static NormalizedRect convertToNormalizedRect(
ImageProcessingOptions imageProcessingOptions) {
RectF regionOfInterest =
imageProcessingOptions.regionOfInterest().isPresent()
? imageProcessingOptions.regionOfInterest().get()
: new RectF(0, 0, 1, 1);
return NormalizedRect.newBuilder()
.setXCenter(rect.centerX())
.setYCenter(rect.centerY())
.setWidth(rect.width())
.setHeight(rect.height())
.setXCenter(regionOfInterest.centerX())
.setYCenter(regionOfInterest.centerY())
.setWidth(regionOfInterest.width())
.setHeight(regionOfInterest.height())
// Convert to radians anti-clockwise.
.setRotation(-(float) Math.PI * imageProcessingOptions.rotationDegrees() / 180.0f)
.build();
}
}
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/core/ImageProcessingOptions.java
+92
View File
@@ -0,0 +1,92 @@
// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.tasks.vision.core;
import android.graphics.RectF;
import com.google.auto.value.AutoValue;
import java.util.Optional;
// TODO: add support for image flipping.
/** Options for image processing. */
@AutoValue
public abstract class ImageProcessingOptions {
/**
* Builder for {@link ImageProcessingOptions}.
*
* <p>If both region-of-interest and rotation are specified, the crop around the
* region-of-interest is extracted first, then the specified rotation is applied to the crop.
*/
@AutoValue.Builder
public abstract static class Builder {
/**
* Sets the optional region-of-interest to crop from the image. If not specified, the full image
* is used.
*
* <p>Coordinates must be in [0,1], {@code left} must be < {@code right} and {@code top} must be
* < {@code bottom}, otherwise an IllegalArgumentException will be thrown when {@link #build()}
* is called.
*/
public abstract Builder setRegionOfInterest(RectF value);
/**
* Sets the rotation to apply to the image (or cropped region-of-interest), in degrees
* clockwise. Defaults to 0.
*
* <p>The rotation must be a multiple (positive or negative) of 90°, otherwise an
* IllegalArgumentException will be thrown when {@link #build()} is called.
*/
public abstract Builder setRotationDegrees(int value);
abstract ImageProcessingOptions autoBuild();
/**
* Validates and builds the {@link ImageProcessingOptions} instance.
*
* @throws IllegalArgumentException if some of the provided values do not meet their
* requirements.
*/
public final ImageProcessingOptions build() {
ImageProcessingOptions options = autoBuild();
if (options.regionOfInterest().isPresent()) {
RectF roi = options.regionOfInterest().get();
if (roi.left >= roi.right || roi.top >= roi.bottom) {
throw new IllegalArgumentException(
String.format(
"Expected left < right and top < bottom, found: %s.", roi.toShortString()));
}
if (roi.left < 0 || roi.right > 1 || roi.top < 0 || roi.bottom > 1) {
throw new IllegalArgumentException(
String.format("Expected RectF values in [0,1], found: %s.", roi.toShortString()));
}
}
if (options.rotationDegrees() % 90 != 0) {
throw new IllegalArgumentException(
String.format(
"Expected rotation to be a multiple of 90°, found: %d.",
options.rotationDegrees()));
}
return options;
}
}
public abstract Optional<RectF> regionOfInterest();
public abstract int rotationDegrees();
public static Builder builder() {
return new AutoValue_ImageProcessingOptions.Builder().setRotationDegrees(0);
}
}
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognitionResult.java
+10 -2
View File
@@ -31,6 +31,8 @@ import java.util.List;
@AutoValue
public abstract class GestureRecognitionResult implements TaskResult {
private static final int kGestureDefaultIndex = -1;
/**
* Creates a {@link GestureRecognitionResult} instance from the lists of landmarks, handedness,
* and gestures protobuf messages.
@@ -97,7 +99,9 @@ public abstract class GestureRecognitionResult implements TaskResult {
gestures.add(
Category.create(
classification.getScore(),
classification.getIndex(),
// Gesture index is not used, because the final gesture result comes from multiple
// classifiers.
kGestureDefaultIndex,
classification.getLabel(),
classification.getDisplayName()));
}
@@ -123,6 +127,10 @@ public abstract class GestureRecognitionResult implements TaskResult {
/** Handedness of detected hands. */
public abstract List<List<Category>> handednesses();
/** Recognized hand gestures of detected hands */
/**
* Recognized hand gestures of detected hands. Note that the index of the gesture is always -1,
* because the raw indices from multiple gesture classifiers cannot consolidate to a meaningful
* index.
*/
public abstract List<List<Category>> gestures();
}
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java
+171 -74
View File
@@ -25,8 +25,8 @@ import com.google.mediapipe.framework.AndroidPacketGetter;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.PacketGetter;
import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.Image;
import com.google.mediapipe.tasks.components.processors.proto.ClassifierOptionsProto;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.core.ErrorListener;
import com.google.mediapipe.tasks.core.OutputHandler;
@@ -36,7 +36,9 @@ import com.google.mediapipe.tasks.core.TaskOptions;
import com.google.mediapipe.tasks.core.TaskRunner;
import com.google.mediapipe.tasks.core.proto.BaseOptionsProto;
import com.google.mediapipe.tasks.vision.core.BaseVisionTaskApi;
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
import com.google.mediapipe.tasks.vision.core.RunningMode;
import com.google.mediapipe.tasks.vision.gesturerecognizer.proto.GestureClassifierGraphOptionsProto;
import com.google.mediapipe.tasks.vision.gesturerecognizer.proto.GestureRecognizerGraphOptionsProto;
import com.google.mediapipe.tasks.vision.gesturerecognizer.proto.HandGestureRecognizerGraphOptionsProto;
import com.google.mediapipe.tasks.vision.handdetector.proto.HandDetectorGraphOptionsProto;
@@ -58,7 +60,7 @@ import java.util.Optional;
* Model Maker. See <TODO link to the DevSite documentation page>.
*
* <ul>
* <li>Input image {@link Image}
* <li>Input image {@link MPImage}
* <ul>
* <li>The image that gesture recognition runs on.
* </ul>
@@ -71,8 +73,10 @@ import java.util.Optional;
public final class GestureRecognizer extends BaseVisionTaskApi {
private static final String TAG = GestureRecognizer.class.getSimpleName();
private static final String IMAGE_IN_STREAM_NAME = "image_in";
private static final String NORM_RECT_IN_STREAM_NAME = "norm_rect_in";
private static final List<String> INPUT_STREAMS =
Collections.unmodifiableList(Arrays.asList("IMAGE:" + IMAGE_IN_STREAM_NAME));
Collections.unmodifiableList(
Arrays.asList("IMAGE:" + IMAGE_IN_STREAM_NAME, "NORM_RECT:" + NORM_RECT_IN_STREAM_NAME));
private static final List<String> OUTPUT_STREAMS =
Collections.unmodifiableList(
Arrays.asList(
@@ -148,9 +152,9 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
public static GestureRecognizer createFromOptions(
Context context, GestureRecognizerOptions recognizerOptions) {
// TODO: Consolidate OutputHandler and TaskRunner.
OutputHandler<GestureRecognitionResult, Image> handler = new OutputHandler<>();
OutputHandler<GestureRecognitionResult, MPImage> handler = new OutputHandler<>();
handler.setOutputPacketConverter(
new OutputHandler.OutputPacketConverter<GestureRecognitionResult, Image>() {
new OutputHandler.OutputPacketConverter<GestureRecognitionResult, MPImage>() {
@Override
public GestureRecognitionResult convertToTaskResult(List<Packet> packets) {
// If there is no hands detected in the image, just returns empty lists.
@@ -175,7 +179,7 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
}
@Override
public Image convertToTaskInput(List<Packet> packets) {
public MPImage convertToTaskInput(List<Packet> packets) {
return new BitmapImageBuilder(
AndroidPacketGetter.getBitmapFromRgb(packets.get(IMAGE_OUT_STREAM_INDEX)))
.build();
@@ -205,7 +209,26 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
* @param runningMode a mediapipe vision task {@link RunningMode}.
*/
private GestureRecognizer(TaskRunner taskRunner, RunningMode runningMode) {
super(taskRunner, runningMode, IMAGE_IN_STREAM_NAME);
super(taskRunner, runningMode, IMAGE_IN_STREAM_NAME, NORM_RECT_IN_STREAM_NAME);
}
/**
* Performs gesture recognition on the provided single image with default image processing
* options, i.e. without any rotation applied. Only use this method when the {@link
* GestureRecognizer} is created with {@link RunningMode.IMAGE}. TODO update java doc
* for input image format.
*
* <p>{@link GestureRecognizer} supports the following color space types:
*
* <ul>
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param image a MediaPipe {@link MPImage} object for processing.
* @throws MediaPipeException if there is an internal error.
*/
public GestureRecognitionResult recognize(MPImage image) {
return recognize(image, ImageProcessingOptions.builder().build());
}
/**
@@ -219,11 +242,41 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference. Note that region-of-interest is <b>not</b> supported
* by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
* this method throwing an IllegalArgumentException.
* @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
* region-of-interest.
* @throws MediaPipeException if there is an internal error.
*/
public GestureRecognitionResult recognize(Image inputImage) {
return (GestureRecognitionResult) processImageData(inputImage);
public GestureRecognitionResult recognize(
MPImage image, ImageProcessingOptions imageProcessingOptions) {
validateImageProcessingOptions(imageProcessingOptions);
return (GestureRecognitionResult) processImageData(image, imageProcessingOptions);
}
/**
* Performs gesture recognition on the provided video frame with default image processing options,
* i.e. without any rotation applied. Only use this method when the {@link GestureRecognizer} is
* created with {@link RunningMode.VIDEO}.
*
* <p>It's required to provide the video frame's timestamp (in milliseconds). The input timestamps
* must be monotonically increasing.
*
* <p>{@link GestureRecognizer} supports the following color space types:
*
* <ul>
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param image a MediaPipe {@link MPImage} object for processing.
* @param timestampMs the input timestamp (in milliseconds).
* @throws MediaPipeException if there is an internal error.
*/
public GestureRecognitionResult recognizeForVideo(MPImage image, long timestampMs) {
return recognizeForVideo(image, ImageProcessingOptions.builder().build(), timestampMs);
}
/**
@@ -239,12 +292,43 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param inputTimestampMs the input timestamp (in milliseconds).
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference. Note that region-of-interest is <b>not</b> supported
* by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
* this method throwing an IllegalArgumentException.
* @param timestampMs the input timestamp (in milliseconds).
* @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
* region-of-interest.
* @throws MediaPipeException if there is an internal error.
*/
public GestureRecognitionResult recognizeForVideo(Image inputImage, long inputTimestampMs) {
return (GestureRecognitionResult) processVideoData(inputImage, inputTimestampMs);
public GestureRecognitionResult recognizeForVideo(
MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
validateImageProcessingOptions(imageProcessingOptions);
return (GestureRecognitionResult) processVideoData(image, imageProcessingOptions, timestampMs);
}
/**
* Sends live image data to perform gesture recognition with default image processing options,
* i.e. without any rotation applied, and the results will be available via the {@link
* ResultListener} provided in the {@link GestureRecognizerOptions}. Only use this method when the
* {@link GestureRecognition} is created with {@link RunningMode.LIVE_STREAM}.
*
* <p>It's required to provide a timestamp (in milliseconds) to indicate when the input image is
* sent to the gesture recognizer. The input timestamps must be monotonically increasing.
*
* <p>{@link GestureRecognizer} supports the following color space types:
*
* <ul>
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param image a MediaPipe {@link MPImage} object for processing.
* @param timestampMs the input timestamp (in milliseconds).
* @throws MediaPipeException if there is an internal error.
*/
public void recognizeAsync(MPImage image, long timestampMs) {
recognizeAsync(image, ImageProcessingOptions.builder().build(), timestampMs);
}
/**
@@ -261,12 +345,20 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param inputTimestampMs the input timestamp (in milliseconds).
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference. Note that region-of-interest is <b>not</b> supported
* by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
* this method throwing an IllegalArgumentException.
* @param timestampMs the input timestamp (in milliseconds).
* @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
* region-of-interest.
* @throws MediaPipeException if there is an internal error.
*/
public void recognizeAsync(Image inputImage, long inputTimestampMs) {
sendLiveStreamData(inputImage, inputTimestampMs);
public void recognizeAsync(
MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
validateImageProcessingOptions(imageProcessingOptions);
sendLiveStreamData(image, imageProcessingOptions, timestampMs);
}
/** Options for setting up an {@link GestureRecognizer}. */
@@ -293,40 +385,46 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
*/
public abstract Builder setRunningMode(RunningMode value);
// TODO: remove these. Temporary solutions before bundle asset is ready.
public abstract Builder setBaseOptionsHandDetector(BaseOptions value);
public abstract Builder setBaseOptionsHandLandmarker(BaseOptions value);
public abstract Builder setBaseOptionsGestureRecognizer(BaseOptions value);
/** Sets the maximum number of hands can be detected by the GestureRecognizer. */
public abstract Builder setNumHands(Integer value);
/** Sets minimum confidence score for the hand detection to be considered successfully */
/** Sets minimum confidence score for the hand detection to be considered successful */
public abstract Builder setMinHandDetectionConfidence(Float value);
/** Sets minimum confidence score of hand presence score in the hand landmark detection. */
public abstract Builder setMinHandPresenceConfidence(Float value);
/** Sets the minimum confidence score for the hand tracking to be considered successfully. */
/** Sets the minimum confidence score for the hand tracking to be considered successful. */
public abstract Builder setMinTrackingConfidence(Float value);
/**
* Sets the minimum confidence score for the gestures to be considered successfully. If < 0,
* the gesture confidence threshold=0.5 for the model is used.
* Sets the optional {@link ClassifierOptions} controling the canned gestures classifier, such
* as score threshold, allow list and deny list of gestures. The categories for canned gesture
* classifiers are: ["None", "Closed_Fist", "Open_Palm", "Pointing_Up", "Thumb_Down",
* "Thumb_Up", "Victory", "ILoveYou"]
*
* <p>TODO Note this option is subject to change, after scoring merging
* calculator is implemented.
*/
public abstract Builder setMinGestureConfidence(Float value);
public abstract Builder setCannedGesturesClassifierOptions(
ClassifierOptions classifierOptions);
/**
* Sets the optional {@link ClassifierOptions} controling the custom gestures classifier, such
* as score threshold, allow list and deny list of gestures.
*
* <p>TODO Note this option is subject to change, after scoring merging
* calculator is implemented.
*/
public abstract Builder setCustomGesturesClassifierOptions(
ClassifierOptions classifierOptions);
/**
* Sets the result listener to receive the detection results asynchronously when the gesture
* recognizer is in the live stream mode.
*/
public abstract Builder setResultListener(
ResultListener<GestureRecognitionResult, Image> value);
ResultListener<GestureRecognitionResult, MPImage> value);
/** Sets an optional error listener. */
public abstract Builder setErrorListener(ErrorListener value);
@@ -359,13 +457,6 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
abstract BaseOptions baseOptions();
// TODO: remove these. Temporary solutions before bundle asset is ready.
abstract BaseOptions baseOptionsHandDetector();
abstract BaseOptions baseOptionsHandLandmarker();
abstract BaseOptions baseOptionsGestureRecognizer();
abstract RunningMode runningMode();
abstract Optional<Integer> numHands();
@@ -376,10 +467,11 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
abstract Optional<Float> minTrackingConfidence();
// TODO update gesture confidence options after score merging calculator is ready.
abstract Optional<Float> minGestureConfidence();
abstract Optional<ClassifierOptions> cannedGesturesClassifierOptions();
abstract Optional<ResultListener<GestureRecognitionResult, Image>> resultListener();
abstract Optional<ClassifierOptions> customGesturesClassifierOptions();
abstract Optional<ResultListener<GestureRecognitionResult, MPImage>> resultListener();
abstract Optional<ErrorListener> errorListener();
@@ -389,8 +481,7 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
.setNumHands(1)
.setMinHandDetectionConfidence(0.5f)
.setMinHandPresenceConfidence(0.5f)
.setMinTrackingConfidence(0.5f)
.setMinGestureConfidence(-1f);
.setMinTrackingConfidence(0.5f);
}
/**
@@ -398,22 +489,18 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
*/
@Override
public CalculatorOptions convertToCalculatorOptionsProto() {
BaseOptionsProto.BaseOptions.Builder baseOptionsBuilder =
BaseOptionsProto.BaseOptions.newBuilder()
.setUseStreamMode(runningMode() != RunningMode.IMAGE)
.mergeFrom(convertBaseOptionsToProto(baseOptions()));
GestureRecognizerGraphOptionsProto.GestureRecognizerGraphOptions.Builder taskOptionsBuilder =
GestureRecognizerGraphOptionsProto.GestureRecognizerGraphOptions.newBuilder()
.setBaseOptions(baseOptionsBuilder);
.setBaseOptions(
BaseOptionsProto.BaseOptions.newBuilder()
.setUseStreamMode(runningMode() != RunningMode.IMAGE)
.mergeFrom(convertBaseOptionsToProto(baseOptions()))
.build());
// Setup HandDetectorGraphOptions.
HandDetectorGraphOptionsProto.HandDetectorGraphOptions.Builder
handDetectorGraphOptionsBuilder =
HandDetectorGraphOptionsProto.HandDetectorGraphOptions.newBuilder()
.setBaseOptions(
BaseOptionsProto.BaseOptions.newBuilder()
.setUseStreamMode(runningMode() != RunningMode.IMAGE)
.mergeFrom(convertBaseOptionsToProto(baseOptionsHandDetector())));
HandDetectorGraphOptionsProto.HandDetectorGraphOptions.newBuilder();
numHands().ifPresent(handDetectorGraphOptionsBuilder::setNumHands);
minHandDetectionConfidence()
.ifPresent(handDetectorGraphOptionsBuilder::setMinDetectionConfidence);
@@ -421,19 +508,12 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
// Setup HandLandmarkerGraphOptions.
HandLandmarksDetectorGraphOptionsProto.HandLandmarksDetectorGraphOptions.Builder
handLandmarksDetectorGraphOptionsBuilder =
HandLandmarksDetectorGraphOptionsProto.HandLandmarksDetectorGraphOptions.newBuilder()
.setBaseOptions(
BaseOptionsProto.BaseOptions.newBuilder()
.setUseStreamMode(runningMode() != RunningMode.IMAGE)
.mergeFrom(convertBaseOptionsToProto(baseOptionsHandLandmarker())));
HandLandmarksDetectorGraphOptionsProto.HandLandmarksDetectorGraphOptions.newBuilder();
minHandPresenceConfidence()
.ifPresent(handLandmarksDetectorGraphOptionsBuilder::setMinDetectionConfidence);
HandLandmarkerGraphOptionsProto.HandLandmarkerGraphOptions.Builder
handLandmarkerGraphOptionsBuilder =
HandLandmarkerGraphOptionsProto.HandLandmarkerGraphOptions.newBuilder()
.setBaseOptions(
BaseOptionsProto.BaseOptions.newBuilder()
.setUseStreamMode(runningMode() != RunningMode.IMAGE));
HandLandmarkerGraphOptionsProto.HandLandmarkerGraphOptions.newBuilder();
minTrackingConfidence()
.ifPresent(handLandmarkerGraphOptionsBuilder::setMinTrackingConfidence);
handLandmarkerGraphOptionsBuilder
@@ -443,17 +523,23 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
// Setup HandGestureRecognizerGraphOptions.
HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.Builder
handGestureRecognizerGraphOptionsBuilder =
HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.newBuilder()
.setBaseOptions(
BaseOptionsProto.BaseOptions.newBuilder()
.setUseStreamMode(runningMode() != RunningMode.IMAGE)
.mergeFrom(convertBaseOptionsToProto(baseOptionsGestureRecognizer())));
ClassifierOptionsProto.ClassifierOptions.Builder classifierOptionsBuilder =
ClassifierOptionsProto.ClassifierOptions.newBuilder();
minGestureConfidence().ifPresent(classifierOptionsBuilder::setScoreThreshold);
handGestureRecognizerGraphOptionsBuilder.setClassifierOptions(
classifierOptionsBuilder.build());
HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.newBuilder();
cannedGesturesClassifierOptions()
.ifPresent(
classifierOptions -> {
handGestureRecognizerGraphOptionsBuilder.setCannedGestureClassifierGraphOptions(
GestureClassifierGraphOptionsProto.GestureClassifierGraphOptions.newBuilder()
.setClassifierOptions(classifierOptions.convertToProto())
.build());
});
customGesturesClassifierOptions()
.ifPresent(
classifierOptions -> {
handGestureRecognizerGraphOptionsBuilder.setCustomGestureClassifierGraphOptions(
GestureClassifierGraphOptionsProto.GestureClassifierGraphOptions.newBuilder()
.setClassifierOptions(classifierOptions.convertToProto())
.build());
});
taskOptionsBuilder
.setHandLandmarkerGraphOptions(handLandmarkerGraphOptionsBuilder.build())
.setHandGestureRecognizerGraphOptions(handGestureRecognizerGraphOptionsBuilder.build());
@@ -464,4 +550,15 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
.build();
}
}
/**
* Validates that the provided {@link ImageProcessingOptions} doesn't contain a
* region-of-interest.
*/
private static void validateImageProcessingOptions(
ImageProcessingOptions imageProcessingOptions) {
if (imageProcessingOptions.regionOfInterest().isPresent()) {
throw new IllegalArgumentException("GestureRecognizer doesn't support region-of-interest.");
}
}
}
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageclassifier/ImageClassificationResult.java
+2 -2
View File
@@ -15,11 +15,11 @@
package com.google.mediapipe.tasks.vision.imageclassifier;
import com.google.auto.value.AutoValue;
import com.google.mediapipe.tasks.components.container.proto.CategoryProto;
import com.google.mediapipe.tasks.components.container.proto.ClassificationsProto;
import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.containers.ClassificationEntry;
import com.google.mediapipe.tasks.components.containers.Classifications;
import com.google.mediapipe.tasks.components.containers.proto.CategoryProto;
import com.google.mediapipe.tasks.components.containers.proto.ClassificationsProto;
import com.google.mediapipe.tasks.core.TaskResult;
import java.util.ArrayList;
import java.util.Collections;
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageclassifier/ImageClassifier.java
+68 -70
View File
@@ -15,7 +15,6 @@
package com.google.mediapipe.tasks.vision.imageclassifier;
import android.content.Context;
import android.graphics.RectF;
import android.os.ParcelFileDescriptor;
import com.google.auto.value.AutoValue;
import com.google.mediapipe.proto.CalculatorOptionsProto.CalculatorOptions;
@@ -25,8 +24,8 @@ import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.PacketGetter;
import com.google.mediapipe.framework.ProtoUtil;
import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.Image;
import com.google.mediapipe.tasks.components.container.proto.ClassificationsProto;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.containers.proto.ClassificationsProto;
import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.core.ErrorListener;
@@ -37,9 +36,9 @@ import com.google.mediapipe.tasks.core.TaskOptions;
import com.google.mediapipe.tasks.core.TaskRunner;
import com.google.mediapipe.tasks.core.proto.BaseOptionsProto;
import com.google.mediapipe.tasks.vision.core.BaseVisionTaskApi;
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
import com.google.mediapipe.tasks.vision.core.RunningMode;
import com.google.mediapipe.tasks.vision.imageclassifier.proto.ImageClassifierGraphOptionsProto;
import com.google.protobuf.InvalidProtocolBufferException;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -165,9 +164,9 @@ public final class ImageClassifier extends BaseVisionTaskApi {
* @throws MediaPipeException if there is an error during {@link ImageClassifier} creation.
*/
public static ImageClassifier createFromOptions(Context context, ImageClassifierOptions options) {
OutputHandler<ImageClassificationResult, Image> handler = new OutputHandler<>();
OutputHandler<ImageClassificationResult, MPImage> handler = new OutputHandler<>();
handler.setOutputPacketConverter(
new OutputHandler.OutputPacketConverter<ImageClassificationResult, Image>() {
new OutputHandler.OutputPacketConverter<ImageClassificationResult, MPImage>() {
@Override
public ImageClassificationResult convertToTaskResult(List<Packet> packets) {
try {
@@ -176,14 +175,14 @@ public final class ImageClassifier extends BaseVisionTaskApi {
packets.get(CLASSIFICATION_RESULT_OUT_STREAM_INDEX),
ClassificationsProto.ClassificationResult.getDefaultInstance()),
packets.get(CLASSIFICATION_RESULT_OUT_STREAM_INDEX).getTimestamp());
} catch (InvalidProtocolBufferException e) {
} catch (IOException e) {
throw new MediaPipeException(
MediaPipeException.StatusCode.INTERNAL.ordinal(), e.getMessage());
}
}
@Override
public Image convertToTaskInput(List<Packet> packets) {
public MPImage convertToTaskInput(List<Packet> packets) {
return new BitmapImageBuilder(
AndroidPacketGetter.getBitmapFromRgb(packets.get(IMAGE_OUT_STREAM_INDEX)))
.build();
@@ -216,6 +215,24 @@ public final class ImageClassifier extends BaseVisionTaskApi {
super(taskRunner, runningMode, IMAGE_IN_STREAM_NAME, NORM_RECT_IN_STREAM_NAME);
}
/**
* Performs classification on the provided single image with default image processing options,
* i.e. using the whole image as region-of-interest and without any rotation applied. Only use
* this method when the {@link ImageClassifier} is created with {@link RunningMode.IMAGE}.
*
* <p>{@link ImageClassifier} supports the following color space types:
*
* <ul>
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param image a MediaPipe {@link MPImage} object for processing.
* @throws MediaPipeException if there is an internal error.
*/
public ImageClassificationResult classify(MPImage image) {
return classify(image, ImageProcessingOptions.builder().build());
}
/**
* Performs classification on the provided single image. Only use this method when the {@link
* ImageClassifier} is created with {@link RunningMode.IMAGE}.
@@ -226,16 +243,23 @@ public final class ImageClassifier extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference.
* @throws MediaPipeException if there is an internal error.
*/
public ImageClassificationResult classify(Image inputImage) {
return (ImageClassificationResult) processImageData(inputImage, buildFullImageRectF());
public ImageClassificationResult classify(
MPImage image, ImageProcessingOptions imageProcessingOptions) {
return (ImageClassificationResult) processImageData(image, imageProcessingOptions);
}
/**
* Performs classification on the provided single image and region-of-interest. Only use this
* method when the {@link ImageClassifier} is created with {@link RunningMode.IMAGE}.
* Performs classification on the provided video frame with default image processing options, i.e.
* using the whole image as region-of-interest and without any rotation applied. Only use this
* method when the {@link ImageClassifier} is created with {@link RunningMode.VIDEO}.
*
* <p>It's required to provide the video frame's timestamp (in milliseconds). The input timestamps
* must be monotonically increasing.
*
* <p>{@link ImageClassifier} supports the following color space types:
*
@@ -243,13 +267,12 @@ public final class ImageClassifier extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param roi a {@link RectF} specifying the region of interest on which to perform
* classification. Coordinates are expected to be specified as normalized values in [0,1].
* @param image a MediaPipe {@link MPImage} object for processing.
* @param timestampMs the input timestamp (in milliseconds).
* @throws MediaPipeException if there is an internal error.
*/
public ImageClassificationResult classify(Image inputImage, RectF roi) {
return (ImageClassificationResult) processImageData(inputImage, roi);
public ImageClassificationResult classifyForVideo(MPImage image, long timestampMs) {
return classifyForVideo(image, ImageProcessingOptions.builder().build(), timestampMs);
}
/**
@@ -265,21 +288,26 @@ public final class ImageClassifier extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param inputTimestampMs the input timestamp (in milliseconds).
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference.
* @param timestampMs the input timestamp (in milliseconds).
* @throws MediaPipeException if there is an internal error.
*/
public ImageClassificationResult classifyForVideo(Image inputImage, long inputTimestampMs) {
return (ImageClassificationResult)
processVideoData(inputImage, buildFullImageRectF(), inputTimestampMs);
public ImageClassificationResult classifyForVideo(
MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
return (ImageClassificationResult) processVideoData(image, imageProcessingOptions, timestampMs);
}
/**
* Performs classification on the provided video frame with additional region-of-interest. Only
* use this method when the {@link ImageClassifier} is created with {@link RunningMode.VIDEO}.
* Sends live image data to perform classification with default image processing options, i.e.
* using the whole image as region-of-interest and without any rotation applied, and the results
* will be available via the {@link ResultListener} provided in the {@link
* ImageClassifierOptions}. Only use this method when the {@link ImageClassifier} is created with
* {@link RunningMode.LIVE_STREAM}.
*
* <p>It's required to provide the video frame's timestamp (in milliseconds). The input timestamps
* must be monotonically increasing.
* <p>It's required to provide a timestamp (in milliseconds) to indicate when the input image is
* sent to the object detector. The input timestamps must be monotonically increasing.
*
* <p>{@link ImageClassifier} supports the following color space types:
*
@@ -287,15 +315,12 @@ public final class ImageClassifier extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param roi a {@link RectF} specifying the region of interest on which to perform
* classification. Coordinates are expected to be specified as normalized values in [0,1].
* @param inputTimestampMs the input timestamp (in milliseconds).
* @param image a MediaPipe {@link MPImage} object for processing.
* @param timestampMs the input timestamp (in milliseconds).
* @throws MediaPipeException if there is an internal error.
*/
public ImageClassificationResult classifyForVideo(
Image inputImage, RectF roi, long inputTimestampMs) {
return (ImageClassificationResult) processVideoData(inputImage, roi, inputTimestampMs);
public void classifyAsync(MPImage image, long timestampMs) {
classifyAsync(image, ImageProcessingOptions.builder().build(), timestampMs);
}
/**
@@ -312,37 +337,15 @@ public final class ImageClassifier extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param inputTimestampMs the input timestamp (in milliseconds).
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference.
* @param timestampMs the input timestamp (in milliseconds).
* @throws MediaPipeException if there is an internal error.
*/
public void classifyAsync(Image inputImage, long inputTimestampMs) {
sendLiveStreamData(inputImage, buildFullImageRectF(), inputTimestampMs);
}
/**
* Sends live image data and additional region-of-interest to perform classification, and the
* results will be available via the {@link ResultListener} provided in the {@link
* ImageClassifierOptions}. Only use this method when the {@link ImageClassifier} is created with
* {@link RunningMode.LIVE_STREAM}.
*
* <p>It's required to provide a timestamp (in milliseconds) to indicate when the input image is
* sent to the object detector. The input timestamps must be monotonically increasing.
*
* <p>{@link ImageClassifier} supports the following color space types:
*
* <ul>
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param roi a {@link RectF} specifying the region of interest on which to perform
* classification. Coordinates are expected to be specified as normalized values in [0,1].
* @param inputTimestampMs the input timestamp (in milliseconds).
* @throws MediaPipeException if there is an internal error.
*/
public void classifyAsync(Image inputImage, RectF roi, long inputTimestampMs) {
sendLiveStreamData(inputImage, roi, inputTimestampMs);
public void classifyAsync(
MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
sendLiveStreamData(image, imageProcessingOptions, timestampMs);
}
/** Options for setting up and {@link ImageClassifier}. */
@@ -380,7 +383,7 @@ public final class ImageClassifier extends BaseVisionTaskApi {
* the image classifier is in the live stream mode.
*/
public abstract Builder setResultListener(
ResultListener<ImageClassificationResult, Image> resultListener);
ResultListener<ImageClassificationResult, MPImage> resultListener);
/** Sets an optional {@link ErrorListener}. */
public abstract Builder setErrorListener(ErrorListener errorListener);
@@ -417,7 +420,7 @@ public final class ImageClassifier extends BaseVisionTaskApi {
abstract Optional<ClassifierOptions> classifierOptions();
abstract Optional<ResultListener<ImageClassificationResult, Image>> resultListener();
abstract Optional<ResultListener<ImageClassificationResult, MPImage>> resultListener();
abstract Optional<ErrorListener> errorListener();
@@ -448,9 +451,4 @@ public final class ImageClassifier extends BaseVisionTaskApi {
.build();
}
}
/** Creates a RectF covering the full image. */
private static RectF buildFullImageRectF() {
return new RectF(0, 0, 1, 1);
}
}
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java
+121 -19
View File
@@ -22,7 +22,7 @@ import com.google.mediapipe.framework.AndroidPacketGetter;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.PacketGetter;
import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.Image;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.core.ErrorListener;
import com.google.mediapipe.tasks.core.OutputHandler;
@@ -32,6 +32,7 @@ import com.google.mediapipe.tasks.core.TaskOptions;
import com.google.mediapipe.tasks.core.TaskRunner;
import com.google.mediapipe.tasks.core.proto.BaseOptionsProto;
import com.google.mediapipe.tasks.vision.core.BaseVisionTaskApi;
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
import com.google.mediapipe.tasks.vision.core.RunningMode;
import com.google.mediapipe.tasks.vision.objectdetector.proto.ObjectDetectorOptionsProto;
import com.google.mediapipe.formats.proto.DetectionProto.Detection;
@@ -96,8 +97,10 @@ import java.util.Optional;
public final class ObjectDetector extends BaseVisionTaskApi {
private static final String TAG = ObjectDetector.class.getSimpleName();
private static final String IMAGE_IN_STREAM_NAME = "image_in";
private static final String NORM_RECT_IN_STREAM_NAME = "norm_rect_in";
private static final List<String> INPUT_STREAMS =
Collections.unmodifiableList(Arrays.asList("IMAGE:" + IMAGE_IN_STREAM_NAME));
Collections.unmodifiableList(
Arrays.asList("IMAGE:" + IMAGE_IN_STREAM_NAME, "NORM_RECT:" + NORM_RECT_IN_STREAM_NAME));
private static final List<String> OUTPUT_STREAMS =
Collections.unmodifiableList(Arrays.asList("DETECTIONS:detections_out", "IMAGE:image_out"));
private static final int DETECTIONS_OUT_STREAM_INDEX = 0;
@@ -162,9 +165,9 @@ public final class ObjectDetector extends BaseVisionTaskApi {
public static ObjectDetector createFromOptions(
Context context, ObjectDetectorOptions detectorOptions) {
// TODO: Consolidate OutputHandler and TaskRunner.
OutputHandler<ObjectDetectionResult, Image> handler = new OutputHandler<>();
OutputHandler<ObjectDetectionResult, MPImage> handler = new OutputHandler<>();
handler.setOutputPacketConverter(
new OutputHandler.OutputPacketConverter<ObjectDetectionResult, Image>() {
new OutputHandler.OutputPacketConverter<ObjectDetectionResult, MPImage>() {
@Override
public ObjectDetectionResult convertToTaskResult(List<Packet> packets) {
return ObjectDetectionResult.create(
@@ -174,7 +177,7 @@ public final class ObjectDetector extends BaseVisionTaskApi {
}
@Override
public Image convertToTaskInput(List<Packet> packets) {
public MPImage convertToTaskInput(List<Packet> packets) {
return new BitmapImageBuilder(
AndroidPacketGetter.getBitmapFromRgb(packets.get(IMAGE_OUT_STREAM_INDEX)))
.build();
@@ -204,7 +207,25 @@ public final class ObjectDetector extends BaseVisionTaskApi {
* @param runningMode a mediapipe vision task {@link RunningMode}.
*/
private ObjectDetector(TaskRunner taskRunner, RunningMode runningMode) {
super(taskRunner, runningMode, IMAGE_IN_STREAM_NAME);
super(taskRunner, runningMode, IMAGE_IN_STREAM_NAME, NORM_RECT_IN_STREAM_NAME);
}
/**
* Performs object detection on the provided single image with default image processing options,
* i.e. without any rotation applied. Only use this method when the {@link ObjectDetector} is
* created with {@link RunningMode.IMAGE}.
*
* <p>{@link ObjectDetector} supports the following color space types:
*
* <ul>
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param image a MediaPipe {@link MPImage} object for processing.
* @throws MediaPipeException if there is an internal error.
*/
public ObjectDetectionResult detect(MPImage image) {
return detect(image, ImageProcessingOptions.builder().build());
}
/**
@@ -217,11 +238,41 @@ public final class ObjectDetector extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference. Note that region-of-interest is <b>not</b> supported
* by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
* this method throwing an IllegalArgumentException.
* @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
* region-of-interest.
* @throws MediaPipeException if there is an internal error.
*/
public ObjectDetectionResult detect(Image inputImage) {
return (ObjectDetectionResult) processImageData(inputImage);
public ObjectDetectionResult detect(
MPImage image, ImageProcessingOptions imageProcessingOptions) {
validateImageProcessingOptions(imageProcessingOptions);
return (ObjectDetectionResult) processImageData(image, imageProcessingOptions);
}
/**
* Performs object detection on the provided video frame with default image processing options,
* i.e. without any rotation applied. Only use this method when the {@link ObjectDetector} is
* created with {@link RunningMode.VIDEO}.
*
* <p>It's required to provide the video frame's timestamp (in milliseconds). The input timestamps
* must be monotonically increasing.
*
* <p>{@link ObjectDetector} supports the following color space types:
*
* <ul>
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param image a MediaPipe {@link MPImage} object for processing.
* @param timestampMs the input timestamp (in milliseconds).
* @throws MediaPipeException if there is an internal error.
*/
public ObjectDetectionResult detectForVideo(MPImage image, long timestampMs) {
return detectForVideo(image, ImageProcessingOptions.builder().build(), timestampMs);
}
/**
@@ -237,12 +288,43 @@ public final class ObjectDetector extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param inputTimestampMs the input timestamp (in milliseconds).
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference. Note that region-of-interest is <b>not</b> supported
* by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
* this method throwing an IllegalArgumentException.
* @param timestampMs the input timestamp (in milliseconds).
* @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
* region-of-interest.
* @throws MediaPipeException if there is an internal error.
*/
public ObjectDetectionResult detectForVideo(Image inputImage, long inputTimestampMs) {
return (ObjectDetectionResult) processVideoData(inputImage, inputTimestampMs);
public ObjectDetectionResult detectForVideo(
MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
validateImageProcessingOptions(imageProcessingOptions);
return (ObjectDetectionResult) processVideoData(image, imageProcessingOptions, timestampMs);
}
/**
* Sends live image data to perform object detection with default image processing options, i.e.
* without any rotation applied, and the results will be available via the {@link ResultListener}
* provided in the {@link ObjectDetectorOptions}. Only use this method when the {@link
* ObjectDetector} is created with {@link RunningMode.LIVE_STREAM}.
*
* <p>It's required to provide a timestamp (in milliseconds) to indicate when the input image is
* sent to the object detector. The input timestamps must be monotonically increasing.
*
* <p>{@link ObjectDetector} supports the following color space types:
*
* <ul>
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param image a MediaPipe {@link MPImage} object for processing.
* @param timestampMs the input timestamp (in milliseconds).
* @throws MediaPipeException if there is an internal error.
*/
public void detectAsync(MPImage image, long timestampMs) {
detectAsync(image, ImageProcessingOptions.builder().build(), timestampMs);
}
/**
@@ -259,12 +341,20 @@ public final class ObjectDetector extends BaseVisionTaskApi {
* <li>{@link Bitmap.Config.ARGB_8888}
* </ul>
*
* @param inputImage a MediaPipe {@link Image} object for processing.
* @param inputTimestampMs the input timestamp (in milliseconds).
* @param image a MediaPipe {@link MPImage} object for processing.
* @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
* input image before running inference. Note that region-of-interest is <b>not</b> supported
* by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
* this method throwing an IllegalArgumentException.
* @param timestampMs the input timestamp (in milliseconds).
* @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
* region-of-interest.
* @throws MediaPipeException if there is an internal error.
*/
public void detectAsync(Image inputImage, long inputTimestampMs) {
sendLiveStreamData(inputImage, inputTimestampMs);
public void detectAsync(
MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
validateImageProcessingOptions(imageProcessingOptions);
sendLiveStreamData(image, imageProcessingOptions, timestampMs);
}
/** Options for setting up an {@link ObjectDetector}. */
@@ -333,7 +423,8 @@ public final class ObjectDetector extends BaseVisionTaskApi {
* Sets the {@link ResultListener} to receive the detection results asynchronously when the
* object detector is in the live stream mode.
*/
public abstract Builder setResultListener(ResultListener<ObjectDetectionResult, Image> value);
public abstract Builder setResultListener(
ResultListener<ObjectDetectionResult, MPImage> value);
/** Sets an optional {@link ErrorListener}}. */
public abstract Builder setErrorListener(ErrorListener value);
@@ -378,7 +469,7 @@ public final class ObjectDetector extends BaseVisionTaskApi {
abstract List<String> categoryDenylist();
abstract Optional<ResultListener<ObjectDetectionResult, Image>> resultListener();
abstract Optional<ResultListener<ObjectDetectionResult, MPImage>> resultListener();
abstract Optional<ErrorListener> errorListener();
@@ -414,4 +505,15 @@ public final class ObjectDetector extends BaseVisionTaskApi {
.build();
}
}
/**
* Validates that the provided {@link ImageProcessingOptions} doesn't contain a
* region-of-interest.
*/
private static void validateImageProcessingOptions(
ImageProcessingOptions imageProcessingOptions) {
if (imageProcessingOptions.regionOfInterest().isPresent()) {
throw new IllegalArgumentException("ObjectDetector doesn't support region-of-interest.");
}
}
}
mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/core/AndroidManifest.xml
+24
View File
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.google.mediapipe.tasks.vision.coretest"
android:versionCode="1"
android:versionName="1.0" >
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
<uses-sdk android:minSdkVersion="24"
android:targetSdkVersion="30" />
<application
android:label="coretest"
android:name="android.support.multidex.MultiDexApplication"
android:taskAffinity="">
<uses-library android:name="android.test.runner" />
</application>
<instrumentation
android:name="com.google.android.apps.common.testing.testrunner.GoogleInstrumentationTestRunner"
android:targetPackage="com.google.mediapipe.tasks.vision.coretest" />
</manifest>
mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/core/BUILD
+19
View File
@@ -0,0 +1,19 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
# TODO: Enable this in OSS
mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/core/ImageProcessingOptionsTest.java
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.tasks.vision.core;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import android.graphics.RectF;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Test for {@link ImageProcessingOptions}/ */
@RunWith(AndroidJUnit4.class)
public final class ImageProcessingOptionsTest {
@Test
public void succeedsWithValidInputs() throws Exception {
ImageProcessingOptions options =
ImageProcessingOptions.builder()
.setRegionOfInterest(new RectF(0.0f, 0.1f, 1.0f, 0.9f))
.setRotationDegrees(270)
.build();
}
@Test
public void failsWithLeftHigherThanRight() {
IllegalArgumentException exception =
assertThrows(
IllegalArgumentException.class,
() ->
ImageProcessingOptions.builder()
.setRegionOfInterest(new RectF(0.9f, 0.0f, 0.1f, 1.0f))
.build());
assertThat(exception).hasMessageThat().contains("Expected left < right and top < bottom");
}
@Test
public void failsWithBottomHigherThanTop() {
IllegalArgumentException exception =
assertThrows(
IllegalArgumentException.class,
() ->
ImageProcessingOptions.builder()
.setRegionOfInterest(new RectF(0.0f, 0.9f, 1.0f, 0.1f))
.build());
assertThat(exception).hasMessageThat().contains("Expected left < right and top < bottom");
}
@Test
public void failsWithInvalidRotation() {
IllegalArgumentException exception =
assertThrows(
IllegalArgumentException.class,
() -> ImageProcessingOptions.builder().setRotationDegrees(1).build());
assertThat(exception).hasMessageThat().contains("Expected rotation to be a multiple of 90°");
}
}
mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java
+277 -133
View File
@@ -19,17 +19,20 @@ import static org.junit.Assert.assertThrows;
import android.content.res.AssetManager;
import android.graphics.BitmapFactory;
import android.graphics.RectF;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.truth.Correspondence;
import com.google.mediapipe.formats.proto.ClassificationProto;
import com.google.mediapipe.framework.MediaPipeException;
import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.Image;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.containers.Landmark;
import com.google.mediapipe.tasks.components.containers.proto.LandmarksDetectionResultProto.LandmarksDetectionResult;
import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
import com.google.mediapipe.tasks.vision.core.RunningMode;
import com.google.mediapipe.tasks.vision.gesturerecognizer.GestureRecognizer.GestureRecognizerOptions;
import java.io.InputStream;
@@ -43,20 +46,25 @@ import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
@SuiteClasses({GestureRecognizerTest.General.class, GestureRecognizerTest.RunningModeTest.class})
public class GestureRecognizerTest {
private static final String HAND_DETECTOR_MODEL_FILE = "palm_detection_full.tflite";
private static final String HAND_LANDMARKER_MODEL_FILE = "hand_landmark_full.tflite";
private static final String GESTURE_RECOGNIZER_MODEL_FILE =
"cg_classifier_screen3d_landmark_features_nn_2022_08_04_base_simple_model.tflite";
private static final String GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE = "gesture_recognizer.task";
private static final String GESTURE_RECOGNIZER_WITH_CUSTOM_CLASSIFIER_BUNDLE_ASSET_FILE =
"gesture_recognizer_with_custom_classifier.task";
private static final String TWO_HANDS_IMAGE = "right_hands.jpg";
private static final String THUMB_UP_IMAGE = "thumb_up.jpg";
private static final String POINTING_UP_ROTATED_IMAGE = "pointing_up_rotated.jpg";
private static final String NO_HANDS_IMAGE = "cats_and_dogs.jpg";
private static final String FIST_IMAGE = "fist.jpg";
private static final String THUMB_UP_LANDMARKS = "thumb_up_landmarks.pb";
private static final String FIST_LANDMARKS = "fist_landmarks.pb";
private static final String TAG = "Gesture Recognizer Test";
private static final String THUMB_UP_LABEL = "Thumb_Up";
private static final int THUMB_UP_INDEX = 5;
private static final String POINTING_UP_LABEL = "Pointing_Up";
private static final String FIST_LABEL = "Closed_Fist";
private static final String ROCK_LABEL = "Rock";
private static final float LANDMARKS_ERROR_TOLERANCE = 0.03f;
private static final int IMAGE_WIDTH = 382;
private static final int IMAGE_HEIGHT = 406;
private static final int GESTURE_EXPECTED_INDEX = -1;
@RunWith(AndroidJUnit4.class)
public static final class General extends GestureRecognizerTest {
@@ -66,20 +74,16 @@ public class GestureRecognizerTest {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(THUMB_UP_IMAGE));
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL, THUMB_UP_INDEX);
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
@@ -88,13 +92,9 @@ public class GestureRecognizerTest {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
@@ -107,27 +107,22 @@ public class GestureRecognizerTest {
}
@Test
public void recognize_successWithMinGestureConfidence() throws Exception {
public void recognize_successWithScoreThreshold() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
// TODO update the confidence to be in range [0,1] after embedding model
// and scoring calculator is integrated.
.setMinGestureConfidence(3.0f)
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder().setScoreThreshold(0.5f).build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(THUMB_UP_IMAGE));
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL, THUMB_UP_INDEX);
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
// Only contains one top scoring gesture.
assertThat(actualResult.gestures().get(0)).hasSize(1);
assertActualGestureEqualExpectedGesture(
@@ -139,13 +134,9 @@ public class GestureRecognizerTest {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(2)
.build();
GestureRecognizer gestureRecognizer =
@@ -154,6 +145,198 @@ public class GestureRecognizerTest {
gestureRecognizer.recognize(getImageFromAsset(TWO_HANDS_IMAGE));
assertThat(actualResult.handednesses()).hasSize(2);
}
@Test
public void recognize_successWithRotation() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
ImageProcessingOptions imageProcessingOptions =
ImageProcessingOptions.builder().setRotationDegrees(-90).build();
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(
getImageFromAsset(POINTING_UP_ROTATED_IMAGE), imageProcessingOptions);
assertThat(actualResult.gestures()).hasSize(1);
assertThat(actualResult.gestures().get(0).get(0).categoryName()).isEqualTo(POINTING_UP_LABEL);
}
@Test
public void recognize_successWithCannedGestureFist() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
@Test
public void recognize_successWithCustomGestureRock() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(
GESTURE_RECOGNIZER_WITH_CUSTOM_CLASSIFIER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(FIST_LANDMARKS, ROCK_LABEL);
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
@Test
public void recognize_successWithAllowGestureFist() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder()
.setScoreThreshold(0.5f)
.setCategoryAllowlist(Arrays.asList("Closed_Fist"))
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
@Test
public void recognize_successWithDenyGestureFist() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder()
.setScoreThreshold(0.5f)
.setCategoryDenylist(Arrays.asList("Closed_Fist"))
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
assertThat(actualResult.landmarks()).isEmpty();
assertThat(actualResult.worldLandmarks()).isEmpty();
assertThat(actualResult.handednesses()).isEmpty();
assertThat(actualResult.gestures()).isEmpty();
}
@Test
public void recognize_successWithAllowAllGestureExceptFist() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder()
.setScoreThreshold(0.5f)
.setCategoryAllowlist(
Arrays.asList(
"None",
"Open_Palm",
"Pointing_Up",
"Thumb_Down",
"Thumb_Up",
"Victory",
"ILoveYou"))
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
assertThat(actualResult.landmarks()).isEmpty();
assertThat(actualResult.worldLandmarks()).isEmpty();
assertThat(actualResult.handednesses()).isEmpty();
assertThat(actualResult.gestures()).isEmpty();
}
@Test
public void recognize_successWithPreferAlowListThanDenyList() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder()
.setScoreThreshold(0.5f)
.setCategoryAllowlist(Arrays.asList("Closed_Fist"))
.setCategoryDenylist(Arrays.asList("Closed_Fist"))
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
@Test
public void recognize_failsWithRegionOfInterest() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
ImageProcessingOptions imageProcessingOptions =
ImageProcessingOptions.builder().setRegionOfInterest(new RectF(0, 0, 1, 1)).build();
IllegalArgumentException exception =
assertThrows(
IllegalArgumentException.class,
() ->
gestureRecognizer.recognize(
getImageFromAsset(THUMB_UP_IMAGE), imageProcessingOptions));
assertThat(exception)
.hasMessageThat()
.contains("GestureRecognizer doesn't support region-of-interest");
}
}
@RunWith(AndroidJUnit4.class)
@@ -168,19 +351,7 @@ public class GestureRecognizerTest {
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE)
.build())
.setBaseOptionsHandDetector(
BaseOptions.builder()
.setModelAssetPath(HAND_DETECTOR_MODEL_FILE)
.build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder()
.setModelAssetPath(HAND_LANDMARKER_MODEL_FILE)
.build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE)
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setRunningMode(mode)
.setResultListener((gestureRecognitionResult, inputImage) -> {})
@@ -201,15 +372,7 @@ public class GestureRecognizerTest {
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE)
.build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE)
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setRunningMode(RunningMode.LIVE_STREAM)
.build());
@@ -223,13 +386,9 @@ public class GestureRecognizerTest {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setRunningMode(RunningMode.IMAGE)
.build();
@@ -238,12 +397,16 @@ public class GestureRecognizerTest {
MediaPipeException exception =
assertThrows(
MediaPipeException.class,
() -> gestureRecognizer.recognizeForVideo(getImageFromAsset(THUMB_UP_IMAGE), 0));
() ->
gestureRecognizer.recognizeForVideo(
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
exception =
assertThrows(
MediaPipeException.class,
() -> gestureRecognizer.recognizeAsync(getImageFromAsset(THUMB_UP_IMAGE), 0));
() ->
gestureRecognizer.recognizeAsync(
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
}
@@ -252,13 +415,9 @@ public class GestureRecognizerTest {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setRunningMode(RunningMode.VIDEO)
.build();
@@ -272,7 +431,9 @@ public class GestureRecognizerTest {
exception =
assertThrows(
MediaPipeException.class,
() -> gestureRecognizer.recognizeAsync(getImageFromAsset(THUMB_UP_IMAGE), 0));
() ->
gestureRecognizer.recognizeAsync(
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
}
@@ -281,13 +442,9 @@ public class GestureRecognizerTest {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setRunningMode(RunningMode.LIVE_STREAM)
.setResultListener((gestureRecognitionResult, inputImage) -> {})
.build();
@@ -302,7 +459,9 @@ public class GestureRecognizerTest {
exception =
assertThrows(
MediaPipeException.class,
() -> gestureRecognizer.recognizeForVideo(getImageFromAsset(THUMB_UP_IMAGE), 0));
() ->
gestureRecognizer.recognizeForVideo(
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
}
@@ -311,13 +470,9 @@ public class GestureRecognizerTest {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setRunningMode(RunningMode.IMAGE)
.build();
@@ -326,7 +481,7 @@ public class GestureRecognizerTest {
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(THUMB_UP_IMAGE));
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL, THUMB_UP_INDEX);
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
@@ -335,41 +490,34 @@ public class GestureRecognizerTest {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setRunningMode(RunningMode.VIDEO)
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL, THUMB_UP_INDEX);
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
for (int i = 0; i < 3; i++) {
GestureRecognitionResult actualResult =
gestureRecognizer.recognizeForVideo(getImageFromAsset(THUMB_UP_IMAGE), i);
gestureRecognizer.recognizeForVideo(
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ i);
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
}
@Test
public void recognize_failsWithOutOfOrderInputTimestamps() throws Exception {
Image image = getImageFromAsset(THUMB_UP_IMAGE);
MPImage image = getImageFromAsset(THUMB_UP_IMAGE);
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL, THUMB_UP_INDEX);
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setRunningMode(RunningMode.LIVE_STREAM)
.setResultListener(
(actualResult, inputImage) -> {
@@ -380,9 +528,11 @@ public class GestureRecognizerTest {
.build();
try (GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
gestureRecognizer.recognizeAsync(image, 1);
gestureRecognizer.recognizeAsync(image, /*timestampsMs=*/ 1);
MediaPipeException exception =
assertThrows(MediaPipeException.class, () -> gestureRecognizer.recognizeAsync(image, 0));
assertThrows(
MediaPipeException.class,
() -> gestureRecognizer.recognizeAsync(image, /*timestampsMs=*/ 0));
assertThat(exception)
.hasMessageThat()
.contains("having a smaller timestamp than the processed timestamp");
@@ -391,19 +541,15 @@ public class GestureRecognizerTest {
@Test
public void recognize_successWithLiveSteamMode() throws Exception {
Image image = getImageFromAsset(THUMB_UP_IMAGE);
MPImage image = getImageFromAsset(THUMB_UP_IMAGE);
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL, THUMB_UP_INDEX);
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
.setBaseOptionsHandDetector(
BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build())
.setBaseOptionsHandLandmarker(
BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build())
.setBaseOptionsGestureRecognizer(
BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build())
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setRunningMode(RunningMode.LIVE_STREAM)
.setResultListener(
(actualResult, inputImage) -> {
@@ -415,19 +561,19 @@ public class GestureRecognizerTest {
try (GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
for (int i = 0; i < 3; i++) {
gestureRecognizer.recognizeAsync(image, i);
gestureRecognizer.recognizeAsync(image, /*timestampsMs=*/ i);
}
}
}
private static Image getImageFromAsset(String filePath) throws Exception {
private static MPImage getImageFromAsset(String filePath) throws Exception {
AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets();
InputStream istr = assetManager.open(filePath);
return new BitmapImageBuilder(BitmapFactory.decodeStream(istr)).build();
}
private static GestureRecognitionResult getExpectedGestureRecognitionResult(
String filePath, String gestureLabel, int gestureIndex) throws Exception {
String filePath, String gestureLabel) throws Exception {
AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets();
InputStream istr = assetManager.open(filePath);
LandmarksDetectionResult landmarksDetectionResultProto =
@@ -435,9 +581,7 @@ public class GestureRecognizerTest {
ClassificationProto.ClassificationList gesturesProto =
ClassificationProto.ClassificationList.newBuilder()
.addClassification(
ClassificationProto.Classification.newBuilder()
.setLabel(gestureLabel)
.setIndex(gestureIndex))
ClassificationProto.Classification.newBuilder().setLabel(gestureLabel))
.build();
return GestureRecognitionResult.create(
Arrays.asList(landmarksDetectionResultProto.getLandmarks()),
@@ -483,11 +627,11 @@ public class GestureRecognizerTest {
private static void assertActualGestureEqualExpectedGesture(
Category actualGesture, Category expectedGesture) {
assertThat(actualGesture.index()).isEqualTo(actualGesture.index());
assertThat(expectedGesture.categoryName()).isEqualTo(expectedGesture.categoryName());
assertThat(actualGesture.categoryName()).isEqualTo(expectedGesture.categoryName());
assertThat(actualGesture.index()).isEqualTo(GESTURE_EXPECTED_INDEX);
}
private static void assertImageSizeIsExpected(Image inputImage) {
private static void assertImageSizeIsExpected(MPImage inputImage) {
assertThat(inputImage).isNotNull();
assertThat(inputImage.getWidth()).isEqualTo(IMAGE_WIDTH);
assertThat(inputImage.getHeight()).isEqualTo(IMAGE_HEIGHT);
mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/imageclassifier/ImageClassifierTest.java
+76 -15
View File
@@ -24,11 +24,12 @@ import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.mediapipe.framework.MediaPipeException;
import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.Image;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.core.TestUtils;
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
import com.google.mediapipe.tasks.vision.core.RunningMode;
import com.google.mediapipe.tasks.vision.imageclassifier.ImageClassifier.ImageClassifierOptions;
import java.io.InputStream;
@@ -47,7 +48,9 @@ public class ImageClassifierTest {
private static final String FLOAT_MODEL_FILE = "mobilenet_v2_1.0_224.tflite";
private static final String QUANTIZED_MODEL_FILE = "mobilenet_v1_0.25_224_quant.tflite";
private static final String BURGER_IMAGE = "burger.jpg";
private static final String BURGER_ROTATED_IMAGE = "burger_rotated.jpg";
private static final String MULTI_OBJECTS_IMAGE = "multi_objects.jpg";
private static final String MULTI_OBJECTS_ROTATED_IMAGE = "multi_objects_rotated.jpg";
@RunWith(AndroidJUnit4.class)
public static final class General extends ImageClassifierTest {
@@ -209,13 +212,60 @@ public class ImageClassifierTest {
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options);
// RectF around the soccer ball.
RectF roi = new RectF(0.450f, 0.308f, 0.614f, 0.734f);
ImageProcessingOptions imageProcessingOptions =
ImageProcessingOptions.builder().setRegionOfInterest(roi).build();
ImageClassificationResult results =
imageClassifier.classify(getImageFromAsset(MULTI_OBJECTS_IMAGE), roi);
imageClassifier.classify(getImageFromAsset(MULTI_OBJECTS_IMAGE), imageProcessingOptions);
assertHasOneHeadAndOneTimestamp(results, 0);
assertCategoriesAre(
results, Arrays.asList(Category.create(0.9969325f, 806, "soccer ball", "")));
}
@Test
public void classify_succeedsWithRotation() throws Exception {
ImageClassifierOptions options =
ImageClassifierOptions.builder()
.setBaseOptions(BaseOptions.builder().setModelAssetPath(FLOAT_MODEL_FILE).build())
.setClassifierOptions(ClassifierOptions.builder().setMaxResults(3).build())
.build();
ImageClassifier imageClassifier =
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options);
ImageProcessingOptions imageProcessingOptions =
ImageProcessingOptions.builder().setRotationDegrees(-90).build();
ImageClassificationResult results =
imageClassifier.classify(getImageFromAsset(BURGER_ROTATED_IMAGE), imageProcessingOptions);
assertHasOneHeadAndOneTimestamp(results, 0);
assertCategoriesAre(
results,
Arrays.asList(
Category.create(0.6390683f, 934, "cheeseburger", ""),
Category.create(0.0495407f, 963, "meat loaf", ""),
Category.create(0.0469720f, 925, "guacamole", "")));
}
@Test
public void classify_succeedsWithRegionOfInterestAndRotation() throws Exception {
ImageClassifierOptions options =
ImageClassifierOptions.builder()
.setBaseOptions(BaseOptions.builder().setModelAssetPath(FLOAT_MODEL_FILE).build())
.setClassifierOptions(ClassifierOptions.builder().setMaxResults(1).build())
.build();
ImageClassifier imageClassifier =
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options);
// RectF around the chair.
RectF roi = new RectF(0.0f, 0.1763f, 0.5642f, 0.3049f);
ImageProcessingOptions imageProcessingOptions =
ImageProcessingOptions.builder().setRegionOfInterest(roi).setRotationDegrees(-90).build();
ImageClassificationResult results =
imageClassifier.classify(
getImageFromAsset(MULTI_OBJECTS_ROTATED_IMAGE), imageProcessingOptions);
assertHasOneHeadAndOneTimestamp(results, 0);
assertCategoriesAre(
results, Arrays.asList(Category.create(0.686824f, 560, "folding chair", "")));
}
}
@RunWith(AndroidJUnit4.class)
@@ -269,12 +319,16 @@ public class ImageClassifierTest {
MediaPipeException exception =
assertThrows(
MediaPipeException.class,
() -> imageClassifier.classifyForVideo(getImageFromAsset(BURGER_IMAGE), 0));
() ->
imageClassifier.classifyForVideo(
getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
exception =
assertThrows(
MediaPipeException.class,
() -> imageClassifier.classifyAsync(getImageFromAsset(BURGER_IMAGE), 0));
() ->
imageClassifier.classifyAsync(
getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
}
@@ -296,7 +350,9 @@ public class ImageClassifierTest {
exception =
assertThrows(
MediaPipeException.class,
() -> imageClassifier.classifyAsync(getImageFromAsset(BURGER_IMAGE), 0));
() ->
imageClassifier.classifyAsync(
getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
}
@@ -320,7 +376,9 @@ public class ImageClassifierTest {
exception =
assertThrows(
MediaPipeException.class,
() -> imageClassifier.classifyForVideo(getImageFromAsset(BURGER_IMAGE), 0));
() ->
imageClassifier.classifyForVideo(
getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
}
@@ -342,7 +400,7 @@ public class ImageClassifierTest {
@Test
public void classify_succeedsWithVideoMode() throws Exception {
Image image = getImageFromAsset(BURGER_IMAGE);
MPImage image = getImageFromAsset(BURGER_IMAGE);
ImageClassifierOptions options =
ImageClassifierOptions.builder()
.setBaseOptions(BaseOptions.builder().setModelAssetPath(FLOAT_MODEL_FILE).build())
@@ -352,7 +410,8 @@ public class ImageClassifierTest {
ImageClassifier imageClassifier =
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options);
for (int i = 0; i < 3; i++) {
ImageClassificationResult results = imageClassifier.classifyForVideo(image, i);
ImageClassificationResult results =
imageClassifier.classifyForVideo(image, /*timestampMs=*/ i);
assertHasOneHeadAndOneTimestamp(results, i);
assertCategoriesAre(
results, Arrays.asList(Category.create(0.7952058f, 934, "cheeseburger", "")));
@@ -361,7 +420,7 @@ public class ImageClassifierTest {
@Test
public void classify_failsWithOutOfOrderInputTimestamps() throws Exception {
Image image = getImageFromAsset(BURGER_IMAGE);
MPImage image = getImageFromAsset(BURGER_IMAGE);
ImageClassifierOptions options =
ImageClassifierOptions.builder()
.setBaseOptions(BaseOptions.builder().setModelAssetPath(FLOAT_MODEL_FILE).build())
@@ -377,9 +436,11 @@ public class ImageClassifierTest {
.build();
try (ImageClassifier imageClassifier =
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
imageClassifier.classifyAsync(getImageFromAsset(BURGER_IMAGE), 1);
imageClassifier.classifyAsync(getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 1);
MediaPipeException exception =
assertThrows(MediaPipeException.class, () -> imageClassifier.classifyAsync(image, 0));
assertThrows(
MediaPipeException.class,
() -> imageClassifier.classifyAsync(image, /*timestampMs=*/ 0));
assertThat(exception)
.hasMessageThat()
.contains("having a smaller timestamp than the processed timestamp");
@@ -388,7 +449,7 @@ public class ImageClassifierTest {
@Test
public void classify_succeedsWithLiveStreamMode() throws Exception {
Image image = getImageFromAsset(BURGER_IMAGE);
MPImage image = getImageFromAsset(BURGER_IMAGE);
ImageClassifierOptions options =
ImageClassifierOptions.builder()
.setBaseOptions(BaseOptions.builder().setModelAssetPath(FLOAT_MODEL_FILE).build())
@@ -405,13 +466,13 @@ public class ImageClassifierTest {
try (ImageClassifier imageClassifier =
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
for (int i = 0; i < 3; ++i) {
imageClassifier.classifyAsync(image, i);
imageClassifier.classifyAsync(image, /*timestampMs=*/ i);
}
}
}
}
private static Image getImageFromAsset(String filePath) throws Exception {
private static MPImage getImageFromAsset(String filePath) throws Exception {
AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets();
InputStream istr = assetManager.open(filePath);
return new BitmapImageBuilder(BitmapFactory.decodeStream(istr)).build();
@@ -437,7 +498,7 @@ public class ImageClassifierTest {
}
}
private static void assertImageSizeIsExpected(Image inputImage) {
private static void assertImageSizeIsExpected(MPImage inputImage) {
assertThat(inputImage).isNotNull();
assertThat(inputImage.getWidth()).isEqualTo(480);
assertThat(inputImage.getHeight()).isEqualTo(325);
mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectorTest.java
+74 -21
View File
@@ -24,11 +24,12 @@ import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.mediapipe.framework.MediaPipeException;
import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.Image;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.containers.Detection;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.core.TestUtils;
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
import com.google.mediapipe.tasks.vision.core.RunningMode;
import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetector.ObjectDetectorOptions;
import java.io.InputStream;
@@ -45,10 +46,11 @@ import org.junit.runners.Suite.SuiteClasses;
public class ObjectDetectorTest {
private static final String MODEL_FILE = "coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.tflite";
private static final String CAT_AND_DOG_IMAGE = "cats_and_dogs.jpg";
private static final String CAT_AND_DOG_ROTATED_IMAGE = "cats_and_dogs_rotated.jpg";
private static final int IMAGE_WIDTH = 1200;
private static final int IMAGE_HEIGHT = 600;
private static final float CAT_SCORE = 0.69f;
private static final RectF catBoundingBox = new RectF(611, 164, 986, 596);
private static final RectF CAT_BOUNDING_BOX = new RectF(611, 164, 986, 596);
// TODO: Figure out why android_x86 and android_arm tests have slightly different
// scores (0.6875 vs 0.69921875).
private static final float SCORE_DIFF_TOLERANCE = 0.01f;
@@ -67,7 +69,7 @@ public class ObjectDetectorTest {
ObjectDetector objectDetector =
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE));
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
}
@Test
@@ -104,7 +106,7 @@ public class ObjectDetectorTest {
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE));
// The score threshold should block all other other objects, except cat.
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
}
@Test
@@ -175,7 +177,7 @@ public class ObjectDetectorTest {
ObjectDetector objectDetector =
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE));
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
}
@Test
@@ -228,6 +230,46 @@ public class ObjectDetectorTest {
.contains("`category_allowlist` and `category_denylist` are mutually exclusive options.");
}
@Test
public void detect_succeedsWithRotation() throws Exception {
ObjectDetectorOptions options =
ObjectDetectorOptions.builder()
.setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build())
.setMaxResults(1)
.setCategoryAllowlist(Arrays.asList("cat"))
.build();
ObjectDetector objectDetector =
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
ImageProcessingOptions imageProcessingOptions =
ImageProcessingOptions.builder().setRotationDegrees(-90).build();
ObjectDetectionResult results =
objectDetector.detect(
getImageFromAsset(CAT_AND_DOG_ROTATED_IMAGE), imageProcessingOptions);
assertContainsOnlyCat(results, new RectF(22.0f, 611.0f, 452.0f, 890.0f), 0.7109375f);
}
@Test
public void detect_failsWithRegionOfInterest() throws Exception {
ObjectDetectorOptions options =
ObjectDetectorOptions.builder()
.setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build())
.build();
ObjectDetector objectDetector =
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
ImageProcessingOptions imageProcessingOptions =
ImageProcessingOptions.builder().setRegionOfInterest(new RectF(0, 0, 1, 1)).build();
IllegalArgumentException exception =
assertThrows(
IllegalArgumentException.class,
() ->
objectDetector.detect(
getImageFromAsset(CAT_AND_DOG_IMAGE), imageProcessingOptions));
assertThat(exception)
.hasMessageThat()
.contains("ObjectDetector doesn't support region-of-interest");
}
// TODO: Implement detect_succeedsWithFloatImages, detect_succeedsWithOrientation,
// detect_succeedsWithNumThreads, detect_successWithNumThreadsFromBaseOptions,
// detect_failsWithInvalidNegativeNumThreads, detect_failsWithInvalidNumThreadsAsZero.
@@ -282,12 +324,16 @@ public class ObjectDetectorTest {
MediaPipeException exception =
assertThrows(
MediaPipeException.class,
() -> objectDetector.detectForVideo(getImageFromAsset(CAT_AND_DOG_IMAGE), 0));
() ->
objectDetector.detectForVideo(
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
exception =
assertThrows(
MediaPipeException.class,
() -> objectDetector.detectAsync(getImageFromAsset(CAT_AND_DOG_IMAGE), 0));
() ->
objectDetector.detectAsync(
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
}
@@ -309,7 +355,9 @@ public class ObjectDetectorTest {
exception =
assertThrows(
MediaPipeException.class,
() -> objectDetector.detectAsync(getImageFromAsset(CAT_AND_DOG_IMAGE), 0));
() ->
objectDetector.detectAsync(
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
}
@@ -333,7 +381,9 @@ public class ObjectDetectorTest {
exception =
assertThrows(
MediaPipeException.class,
() -> objectDetector.detectForVideo(getImageFromAsset(CAT_AND_DOG_IMAGE), 0));
() ->
objectDetector.detectForVideo(
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ 0));
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
}
@@ -348,7 +398,7 @@ public class ObjectDetectorTest {
ObjectDetector objectDetector =
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE));
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
}
@Test
@@ -363,30 +413,33 @@ public class ObjectDetectorTest {
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
for (int i = 0; i < 3; i++) {
ObjectDetectionResult results =
objectDetector.detectForVideo(getImageFromAsset(CAT_AND_DOG_IMAGE), i);
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
objectDetector.detectForVideo(
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ i);
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
}
}
@Test
public void detect_failsWithOutOfOrderInputTimestamps() throws Exception {
Image image = getImageFromAsset(CAT_AND_DOG_IMAGE);
MPImage image = getImageFromAsset(CAT_AND_DOG_IMAGE);
ObjectDetectorOptions options =
ObjectDetectorOptions.builder()
.setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build())
.setRunningMode(RunningMode.LIVE_STREAM)
.setResultListener(
(objectDetectionResult, inputImage) -> {
assertContainsOnlyCat(objectDetectionResult, catBoundingBox, CAT_SCORE);
assertContainsOnlyCat(objectDetectionResult, CAT_BOUNDING_BOX, CAT_SCORE);
assertImageSizeIsExpected(inputImage);
})
.setMaxResults(1)
.build();
try (ObjectDetector objectDetector =
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
objectDetector.detectAsync(image, 1);
objectDetector.detectAsync(image, /*timestampsMs=*/ 1);
MediaPipeException exception =
assertThrows(MediaPipeException.class, () -> objectDetector.detectAsync(image, 0));
assertThrows(
MediaPipeException.class,
() -> objectDetector.detectAsync(image, /*timestampsMs=*/ 0));
assertThat(exception)
.hasMessageThat()
.contains("having a smaller timestamp than the processed timestamp");
@@ -395,14 +448,14 @@ public class ObjectDetectorTest {
@Test
public void detect_successWithLiveSteamMode() throws Exception {
Image image = getImageFromAsset(CAT_AND_DOG_IMAGE);
MPImage image = getImageFromAsset(CAT_AND_DOG_IMAGE);
ObjectDetectorOptions options =
ObjectDetectorOptions.builder()
.setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build())
.setRunningMode(RunningMode.LIVE_STREAM)
.setResultListener(
(objectDetectionResult, inputImage) -> {
assertContainsOnlyCat(objectDetectionResult, catBoundingBox, CAT_SCORE);
assertContainsOnlyCat(objectDetectionResult, CAT_BOUNDING_BOX, CAT_SCORE);
assertImageSizeIsExpected(inputImage);
})
.setMaxResults(1)
@@ -410,13 +463,13 @@ public class ObjectDetectorTest {
try (ObjectDetector objectDetector =
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
for (int i = 0; i < 3; i++) {
objectDetector.detectAsync(image, i);
objectDetector.detectAsync(image, /*timestampsMs=*/ i);
}
}
}
}
private static Image getImageFromAsset(String filePath) throws Exception {
private static MPImage getImageFromAsset(String filePath) throws Exception {
AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets();
InputStream istr = assetManager.open(filePath);
return new BitmapImageBuilder(BitmapFactory.decodeStream(istr)).build();
@@ -448,7 +501,7 @@ public class ObjectDetectorTest {
assertThat(boundingBox1.bottom).isWithin(PIXEL_DIFF_TOLERANCE).of(boundingBox2.bottom);
}
private static void assertImageSizeIsExpected(Image inputImage) {
private static void assertImageSizeIsExpected(MPImage inputImage) {
assertThat(inputImage).isNotNull();
assertThat(inputImage.getWidth()).isEqualTo(IMAGE_WIDTH);
assertThat(inputImage.getHeight()).isEqualTo(IMAGE_HEIGHT);
mediapipe/tasks/python/__init__.py
+11
View File
@@ -11,3 +11,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe Tasks API."""
from . import components
from . import core
from . import vision
BaseOptions = core.base_options.BaseOptions
# Remove unnecessary modules to avoid duplication in API docs.
del core
mediapipe/tasks/python/audio/core/BUILD
+37
View File
@@ -0,0 +1,37 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Placeholder for internal Python strict library and test compatibility macro.
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
py_library(
name = "audio_task_running_mode",
srcs = ["audio_task_running_mode.py"],
)
py_library(
name = "base_audio_task_api",
srcs = [
"base_audio_task_api.py",
],
deps = [
":audio_task_running_mode",
"//mediapipe/framework:calculator_py_pb2",
"//mediapipe/python:_framework_bindings",
"//mediapipe/tasks/python/core:optional_dependencies",
],
)
mediapipe/tasks/python/audio/core/__init__.py
+16
View File
@@ -0,0 +1,16 @@
"""Copyright 2022 The MediaPipe Authors.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
mediapipe/tasks/python/audio/core/audio_task_running_mode.py
+29
View File
@@ -0,0 +1,29 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The running mode of MediaPipe Audio Tasks."""
import enum
class AudioTaskRunningMode(enum.Enum):
"""MediaPipe audio task running mode.
Attributes:
AUDIO_CLIPS: The mode for running a mediapipe audio task on independent
audio clips.
AUDIO_STREAM: The mode for running a mediapipe audio task on an audio
stream, such as from microphone.
"""
AUDIO_CLIPS = 'AUDIO_CLIPS'
AUDIO_STREAM = 'AUDIO_STREAM'
mediapipe/tasks/python/audio/core/base_audio_task_api.py
+123
View File
@@ -0,0 +1,123 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe audio task base api."""
from typing import Callable, Mapping, Optional
from mediapipe.framework import calculator_pb2
from mediapipe.python._framework_bindings import packet as packet_module
from mediapipe.python._framework_bindings import task_runner as task_runner_module
from mediapipe.tasks.python.audio.core import audio_task_running_mode as running_mode_module
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
_TaskRunner = task_runner_module.TaskRunner
_Packet = packet_module.Packet
_RunningMode = running_mode_module.AudioTaskRunningMode
class BaseAudioTaskApi(object):
"""The base class of the user-facing mediapipe audio task api classes."""
def __init__(
self,
graph_config: calculator_pb2.CalculatorGraphConfig,
running_mode: _RunningMode,
packet_callback: Optional[Callable[[Mapping[str, packet_module.Packet]],
None]] = None
) -> None:
"""Initializes the `BaseAudioTaskApi` object.
Args:
graph_config: The mediapipe audio task graph config proto.
running_mode: The running mode of the mediapipe audio task.
packet_callback: The optional packet callback for getting results
asynchronously in the audio stream mode.
Raises:
ValueError: The packet callback is not properly set based on the task's
running mode.
"""
if running_mode == _RunningMode.AUDIO_STREAM:
if packet_callback is None:
raise ValueError(
'The audio task is in audio stream mode, a user-defined result '
'callback must be provided.')
elif packet_callback:
raise ValueError(
'The audio task is in audio clips mode, a user-defined result '
'callback should not be provided.')
self._runner = _TaskRunner.create(graph_config, packet_callback)
self._running_mode = running_mode
def _process_audio_clip(
self, inputs: Mapping[str, _Packet]) -> Mapping[str, _Packet]:
"""A synchronous method to process independent audio clips.
The call blocks the current thread until a failure status or a successful
result is returned.
Args:
inputs: A dict contains (input stream name, data packet) pairs.
Returns:
A dict contains (output stream name, data packet) pairs.
Raises:
ValueError: If the task's running mode is not set to audio clips mode.
"""
if self._running_mode != _RunningMode.AUDIO_CLIPS:
raise ValueError(
'Task is not initialized with the audio clips mode. Current running mode:'
+ self._running_mode.name)
return self._runner.process(inputs)
def _send_audio_stream_data(self, inputs: Mapping[str, _Packet]) -> None:
"""An asynchronous method to send audio stream data to the runner.
The results will be available in the user-defined results callback.
Args:
inputs: A dict contains (input stream name, data packet) pairs.
Raises:
ValueError: If the task's running mode is not set to the audio stream
mode.
"""
if self._running_mode != _RunningMode.AUDIO_STREAM:
raise ValueError(
'Task is not initialized with the audio stream mode. Current running mode:'
+ self._running_mode.name)
self._runner.send(inputs)
def close(self) -> None:
"""Shuts down the mediapipe audio task instance.
Raises:
RuntimeError: If the mediapipe audio task failed to close.
"""
self._runner.close()
@doc_controls.do_not_generate_docs
def __enter__(self):
"""Return `self` upon entering the runtime context."""
return self
@doc_controls.do_not_generate_docs
def __exit__(self, unused_exc_type, unused_exc_value, unused_traceback):
"""Shuts down the mediapipe audio task instance on exit of the context manager.
Raises:
RuntimeError: If the mediapipe audio task failed to close.
"""
self.close()

Some files were not shown because too many files have changed in this diff Show More