From d90daa859f34482a2a5232f2367ad4bfe1cf3dc6 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 7 Oct 2022 04:05:09 -0700 Subject: [PATCH] Internal change PiperOrigin-RevId: 479544054 --- .../containers/proto/category.proto | 3 + .../containers/proto/classifications.proto | 3 + .../processors/proto/classifier_options.proto | 3 + .../image_classifier_graph_options.proto | 3 + .../tasks/components/containers/BUILD | 20 +++ .../containers/ClassificationEntry.java | 48 +++++++ .../containers/Classifications.java | 52 ++++++++ .../tasks/components/processors/BUILD | 30 +++++ .../processors/ClassifierOptions.java | 118 ++++++++++++++++++ 9 files changed, 280 insertions(+) create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/ClassificationEntry.java create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Classifications.java create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/BUILD create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/ClassifierOptions.java diff --git a/mediapipe/tasks/cc/components/containers/proto/category.proto b/mediapipe/tasks/cc/components/containers/proto/category.proto index a44fb5b1..a154e5f4 100644 --- a/mediapipe/tasks/cc/components/containers/proto/category.proto +++ b/mediapipe/tasks/cc/components/containers/proto/category.proto @@ -17,6 +17,9 @@ syntax = "proto2"; package mediapipe.tasks.components.containers.proto; +option java_package = "com.google.mediapipe.tasks.components.container.proto"; +option java_outer_classname = "CategoryProto"; + // A single classification result. message Category { // The index of the category in the corresponding label map, usually packed in diff --git a/mediapipe/tasks/cc/components/containers/proto/classifications.proto b/mediapipe/tasks/cc/components/containers/proto/classifications.proto index e0ccad7a..0f5086b9 100644 --- a/mediapipe/tasks/cc/components/containers/proto/classifications.proto +++ b/mediapipe/tasks/cc/components/containers/proto/classifications.proto @@ -19,6 +19,9 @@ package mediapipe.tasks.components.containers.proto; import "mediapipe/tasks/cc/components/containers/proto/category.proto"; +option java_package = "com.google.mediapipe.tasks.components.container.proto"; +option java_outer_classname = "ClassificationsProto"; + // List of predicted categories with an optional timestamp. message ClassificationEntry { // The array of predicted categories, usually sorted by descending scores, diff --git a/mediapipe/tasks/cc/components/processors/proto/classifier_options.proto b/mediapipe/tasks/cc/components/processors/proto/classifier_options.proto index 7afbfc14..12ece724 100644 --- a/mediapipe/tasks/cc/components/processors/proto/classifier_options.proto +++ b/mediapipe/tasks/cc/components/processors/proto/classifier_options.proto @@ -17,6 +17,9 @@ syntax = "proto2"; package mediapipe.tasks.components.processors.proto; +option java_package = "com.google.mediapipe.tasks.components.processors.proto"; +option java_outer_classname = "ClassifierOptionsProto"; + // Shared options used by all classification tasks. message ClassifierOptions { // The locale to use for display names specified through the TFLite Model diff --git a/mediapipe/tasks/cc/vision/image_classifier/proto/image_classifier_graph_options.proto b/mediapipe/tasks/cc/vision/image_classifier/proto/image_classifier_graph_options.proto index b307a66b..76315e23 100644 --- a/mediapipe/tasks/cc/vision/image_classifier/proto/image_classifier_graph_options.proto +++ b/mediapipe/tasks/cc/vision/image_classifier/proto/image_classifier_graph_options.proto @@ -21,6 +21,9 @@ import "mediapipe/framework/calculator.proto"; import "mediapipe/tasks/cc/components/processors/proto/classifier_options.proto"; import "mediapipe/tasks/cc/core/proto/base_options.proto"; +option java_package = "com.google.mediapipe.tasks.vision.imageclassifier.proto"; +option java_outer_classname = "ImageClassifierGraphOptionsProto"; + message ImageClassifierGraphOptions { extend mediapipe.CalculatorOptions { optional ImageClassifierGraphOptions ext = 456383383; diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD index 8f3c1539..23e6124c 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD @@ -34,3 +34,23 @@ android_library( "@maven//:com_google_guava_guava", ], ) + +android_library( + name = "classification_entry", + srcs = ["ClassificationEntry.java"], + deps = [ + ":category", + "//third_party:autovalue", + "@maven//:com_google_guava_guava", + ], +) + +android_library( + name = "classifications", + srcs = ["Classifications.java"], + deps = [ + ":classification_entry", + "//third_party:autovalue", + "@maven//:com_google_guava_guava", + ], +) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/ClassificationEntry.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/ClassificationEntry.java new file mode 100644 index 00000000..8fc1daa0 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/ClassificationEntry.java @@ -0,0 +1,48 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.components.containers; + +import com.google.auto.value.AutoValue; +import java.util.Collections; +import java.util.List; + +/** + * Represents a list of predicted categories with an optional timestamp. Typically used as result + * for classification tasks. + */ +@AutoValue +public abstract class ClassificationEntry { + /** + * Creates a {@link ClassificationEntry} instance from a list of {@link Category} and optional + * timestamp. + * + * @param categories the list of {@link Category} objects that contain category name, display + * name, score and label index. + * @param timestampMs the {@link long} representing the timestamp for which these categories were + * obtained. + */ + public static ClassificationEntry create(List categories, long timestampMs) { + return new AutoValue_ClassificationEntry(Collections.unmodifiableList(categories), timestampMs); + } + + /** The list of predicted {@link Category} objects, sorted by descending score. */ + public abstract List categories(); + + /** + * The timestamp (in milliseconds) associated to the classification entry. This is useful for time + * series use cases, e.g. audio classification. + */ + public abstract long timestampMs(); +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Classifications.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Classifications.java new file mode 100644 index 00000000..72657872 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Classifications.java @@ -0,0 +1,52 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.components.containers; + +import com.google.auto.value.AutoValue; +import java.util.Collections; +import java.util.List; + +/** + * Represents the list of classification for a given classifier head. Typically used as a result for + * classification tasks. + */ +@AutoValue +public abstract class Classifications { + + /** + * Creates a {@link Classifications} instance. + * + * @param entries the list of {@link ClassificationEntry} objects containing the predicted + * categories. + * @param headIndex the index of the classifier head. + * @param headName the name of the classifier head. + */ + public static Classifications create( + List entries, int headIndex, String headName) { + return new AutoValue_Classifications( + Collections.unmodifiableList(entries), headIndex, headName); + } + + /** A list of {@link ClassificationEntry} objects. */ + public abstract List entries(); + + /** + * The index of the classifier head these entries refer to. This is useful for multi-head models. + */ + public abstract int headIndex(); + + /** The name of the classifier head, which is the corresponding tensor metadata name. */ + public abstract String headName(); +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/BUILD new file mode 100644 index 00000000..88516d80 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/BUILD @@ -0,0 +1,30 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +android_library( + name = "classifieroptions", + srcs = ["ClassifierOptions.java"], + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], + deps = [ + "//mediapipe/tasks/cc/components/processors/proto:classifier_options_java_proto_lite", + "//third_party:autovalue", + "@maven//:com_google_guava_guava", + ], +) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/ClassifierOptions.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/ClassifierOptions.java new file mode 100644 index 00000000..76da4b44 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/ClassifierOptions.java @@ -0,0 +1,118 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.components.processors; + +import com.google.auto.value.AutoValue; +import com.google.mediapipe.tasks.components.processors.proto.ClassifierOptionsProto; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** Classifier options shared across MediaPipe Java classification tasks. */ +@AutoValue +public abstract class ClassifierOptions { + + /** Builder for {@link ClassifierOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + /** + * Sets the optional locale to use for display names specified through the TFLite Model + * Metadata, if any. + */ + public abstract Builder setDisplayNamesLocale(String locale); + + /** + * Sets the optional maximum number of top-scored classification results to return. + * + *

If not set, all available results are returned. If set, must be > 0. + */ + public abstract Builder setMaxResults(Integer maxResults); + + /** + * Sets the optional score threshold. Results with score below this value are rejected. + * + *

Overrides the score threshold specified in the TFLite Model Metadata, if any. + */ + public abstract Builder setScoreThreshold(Float scoreThreshold); + + /** + * Sets the optional allowlist of category names. + * + *

If non-empty, detection results whose category name is not in this set will be filtered + * out. Duplicate or unknown category names are ignored. Mutually exclusive with {@code + * categoryDenylist}. + */ + public abstract Builder setCategoryAllowlist(List categoryAllowlist); + + /** + * Sets the optional denylist of category names. + * + *

If non-empty, detection results whose category name is in this set will be filtered out. + * Duplicate or unknown category names are ignored. Mutually exclusive with {@code + * categoryAllowlist}. + */ + public abstract Builder setCategoryDenylist(List categoryDenylist); + + abstract ClassifierOptions autoBuild(); + + /** + * Validates and builds the {@link ClassifierOptions} instance. + * + * @throws IllegalArgumentException if {@link maxResults} is set to a value <= 0. + */ + public final ClassifierOptions build() { + ClassifierOptions options = autoBuild(); + if (options.maxResults().isPresent() && options.maxResults().get() <= 0) { + throw new IllegalArgumentException("If specified, maxResults must be > 0"); + } + return options; + } + } + + public abstract Optional displayNamesLocale(); + + public abstract Optional maxResults(); + + public abstract Optional scoreThreshold(); + + public abstract List categoryAllowlist(); + + public abstract List categoryDenylist(); + + public static Builder builder() { + return new AutoValue_ClassifierOptions.Builder() + .setCategoryAllowlist(Collections.emptyList()) + .setCategoryDenylist(Collections.emptyList()); + } + + /** + * Converts a {@link ClassifierOptions} object to a {@link + * ClassifierOptionsProto.ClassifierOptions} protobuf message. + */ + public ClassifierOptionsProto.ClassifierOptions convertToProto() { + ClassifierOptionsProto.ClassifierOptions.Builder builder = + ClassifierOptionsProto.ClassifierOptions.newBuilder(); + displayNamesLocale().ifPresent(builder::setDisplayNamesLocale); + maxResults().ifPresent(builder::setMaxResults); + scoreThreshold().ifPresent(builder::setScoreThreshold); + if (!categoryAllowlist().isEmpty()) { + builder.addAllCategoryAllowlist(categoryAllowlist()); + } + if (!categoryDenylist().isEmpty()) { + builder.addAllCategoryDenylist(categoryDenylist()); + } + return builder.build(); + } +}