Merge branch 'google:master' into image-embedder-python

mediapipe/tasks/cc/audio/audio_classifier/proto/audio_classifier_graph_options.proto

@@ -21,6 +21,9 @@ import "mediapipe/framework/calculator.proto";
 import "mediapipe/tasks/cc/components/processors/proto/classifier_options.proto";
 import "mediapipe/tasks/cc/core/proto/base_options.proto";
 
+option java_package = "com.google.mediapipe.tasks.audio.audioclassifier.proto";
+option java_outer_classname = "AudioClassifierGraphOptionsProto";
+
 message AudioClassifierGraphOptions {
   extend mediapipe.CalculatorOptions {
     optional AudioClassifierGraphOptions ext = 451755788;

mediapipe/tasks/cc/components/containers/BUILD

@@ -21,15 +21,6 @@ cc_library(
     hdrs = ["rect.h"],
 )
 
-cc_library(
-    name = "gesture_recognition_result",
-    hdrs = ["gesture_recognition_result.h"],
-    deps = [
-        "//mediapipe/framework/formats:classification_cc_proto",
-        "//mediapipe/framework/formats:landmark_cc_proto",
-    ],
-)
-
 cc_library(
     name = "category",
     srcs = ["category.cc"],

mediapipe/tasks/cc/vision/gesture_recognizer/BUILD

@@ -124,12 +124,22 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "gesture_recognizer_result",
+    hdrs = ["gesture_recognizer_result.h"],
+    deps = [
+        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+    ],
+)
+
 cc_library(
     name = "gesture_recognizer",
     srcs = ["gesture_recognizer.cc"],
     hdrs = ["gesture_recognizer.h"],
     deps = [
         ":gesture_recognizer_graph",
+        ":gesture_recognizer_result",
         ":hand_gesture_recognizer_graph",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/api2:builder",
@@ -140,7 +150,6 @@ cc_library(
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/tasks/cc:common",
         "//mediapipe/tasks/cc/components:image_preprocessing",
-        "//mediapipe/tasks/cc/components/containers:gesture_recognition_result",
         "//mediapipe/tasks/cc/components/processors:classifier_options",
         "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto",
         "//mediapipe/tasks/cc/core:base_options",

mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.cc

@@ -58,8 +58,6 @@ namespace {
 using GestureRecognizerGraphOptionsProto = ::mediapipe::tasks::vision::
     gesture_recognizer::proto::GestureRecognizerGraphOptions;
 
-using ::mediapipe::tasks::components::containers::GestureRecognitionResult;
-
 constexpr char kHandGestureSubgraphTypeName[] =
     "mediapipe.tasks.vision.gesture_recognizer.GestureRecognizerGraph";
 
@@ -214,7 +212,7 @@ absl::StatusOr<std::unique_ptr<GestureRecognizer>> GestureRecognizer::Create(
       std::move(packets_callback));
 }
 
-absl::StatusOr<GestureRecognitionResult> GestureRecognizer::Recognize(
+absl::StatusOr<GestureRecognizerResult> GestureRecognizer::Recognize(
     mediapipe::Image image,
     std::optional<core::ImageProcessingOptions> image_processing_options) {
   if (image.UsesGpu()) {
@@ -250,7 +248,7 @@ absl::StatusOr<GestureRecognitionResult> GestureRecognizer::Recognize(
   };
 }
 
-absl::StatusOr<GestureRecognitionResult> GestureRecognizer::RecognizeForVideo(
+absl::StatusOr<GestureRecognizerResult> GestureRecognizer::RecognizeForVideo(
     mediapipe::Image image, int64 timestamp_ms,
     std::optional<core::ImageProcessingOptions> image_processing_options) {
   if (image.UsesGpu()) {

mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.h

@@ -24,12 +24,12 @@ limitations under the License.
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
-#include "mediapipe/tasks/cc/components/containers/gesture_recognition_result.h"
 #include "mediapipe/tasks/cc/components/processors/classifier_options.h"
 #include "mediapipe/tasks/cc/core/base_options.h"
 #include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h"
 #include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
 #include "mediapipe/tasks/cc/vision/core/running_mode.h"
+#include "mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_result.h"
 
 namespace mediapipe {
 namespace tasks {
@@ -81,9 +81,8 @@ struct GestureRecognizerOptions {
   // The user-defined result callback for processing live stream data.
   // The result callback should only be specified when the running mode is set
   // to RunningMode::LIVE_STREAM.
-  std::function<void(
-      absl::StatusOr<components::containers::GestureRecognitionResult>,
-      const Image&, int64)>
+  std::function<void(absl::StatusOr<GestureRecognizerResult>, const Image&,
+                     int64)>
       result_callback = nullptr;
 };
 
@@ -104,7 +103,7 @@ struct GestureRecognizerOptions {
 //       'width' and 'height' fields is NOT supported and will result in an
 //       invalid argument error being returned.
 // Outputs:
-//   GestureRecognitionResult
+//   GestureRecognizerResult
 //     - The hand gesture recognition results.
 class GestureRecognizer : tasks::vision::core::BaseVisionTaskApi {
  public:
@@ -139,7 +138,7 @@ class GestureRecognizer : tasks::vision::core::BaseVisionTaskApi {
   // The image can be of any size with format RGB or RGBA.
   // TODO: Describes how the input image will be preprocessed
   // after the yuv support is implemented.
-  absl::StatusOr<components::containers::GestureRecognitionResult> Recognize(
+  absl::StatusOr<GestureRecognizerResult> Recognize(
       Image image,
       std::optional<core::ImageProcessingOptions> image_processing_options =
           std::nullopt);
@@ -157,10 +156,10 @@ class GestureRecognizer : tasks::vision::core::BaseVisionTaskApi {
   // The image can be of any size with format RGB or RGBA. It's required to
   // provide the video frame's timestamp (in milliseconds). The input timestamps
   // must be monotonically increasing.
-  absl::StatusOr<components::containers::GestureRecognitionResult>
-  RecognizeForVideo(Image image, int64 timestamp_ms,
-                    std::optional<core::ImageProcessingOptions>
-                        image_processing_options = std::nullopt);
+  absl::StatusOr<GestureRecognizerResult> RecognizeForVideo(
+      Image image, int64 timestamp_ms,
+      std::optional<core::ImageProcessingOptions> image_processing_options =
+          std::nullopt);
 
   // Sends live image data to perform gesture recognition, and the results will
   // be available via the "result_callback" provided in the
@@ -179,7 +178,7 @@ class GestureRecognizer : tasks::vision::core::BaseVisionTaskApi {
   // and will result in an invalid argument error being returned.
   //
   // The "result_callback" provides
-  //   - A vector of GestureRecognitionResult, each is the recognized results
+  //   - A vector of GestureRecognizerResult, each is the recognized results
   //     for a input frame.
   //   - The const reference to the corresponding input image that the gesture
   //     recognizer runs on. Note that the const reference to the image will no

mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_result.h

@@ -13,20 +13,20 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_GESTURE_RECOGNITION_RESULT_H_
-#define MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_GESTURE_RECOGNITION_RESULT_H_
+#ifndef MEDIAPIPE_TASKS_CC_VISION_GESTURE_RECOGNIZER_GESTURE_RECOGNIZER_RESULT_H_
+#define MEDIAPIPE_TASKS_CC_VISION_GESTURE_RECOGNIZER_GESTURE_RECOGNIZER_RESULT_H_
 
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 
 namespace mediapipe {
 namespace tasks {
-namespace components {
-namespace containers {
+namespace vision {
+namespace gesture_recognizer {
 
 // The gesture recognition result from GestureRecognizer, where each vector
 // element represents a single hand detected in the image.
-struct GestureRecognitionResult {
+struct GestureRecognizerResult {
   // Recognized hand gestures with sorted order such that the winning label is
   // the first item in the list.
   std::vector<mediapipe::ClassificationList> gestures;
@@ -38,9 +38,9 @@ struct GestureRecognitionResult {
   std::vector<mediapipe::LandmarkList> hand_world_landmarks;
 };
 
-}  // namespace containers
-}  // namespace components
+}  // namespace gesture_recognizer
+}  // namespace vision
 }  // namespace tasks
 }  // namespace mediapipe
 
-#endif  // MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_GESTURE_RECOGNITION_RESULT_H_
+#endif  // MEDIAPIPE_TASKS_CC_VISION_GESTURE_RECOGNIZER_GESTURE_RECOGNIZER_RESULT_H_

mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc

@@ -276,33 +276,44 @@ class HandLandmarkerGraph : public core::ModelTaskGraph {
         .set_min_size(max_num_hands);
     auto has_enough_hands = min_size_node.Out("").Cast<bool>();
 
-    auto image_for_hand_detector =
-        DisallowIf(image_in, has_enough_hands, graph);
-    auto norm_rect_in_for_hand_detector =
-        DisallowIf(norm_rect_in, has_enough_hands, graph);
-
     auto& hand_detector =
         graph.AddNode("mediapipe.tasks.vision.hand_detector.HandDetectorGraph");
     hand_detector.GetOptions<HandDetectorGraphOptions>().CopyFrom(
         tasks_options.hand_detector_graph_options());
-    image_for_hand_detector >> hand_detector.In("IMAGE");
-    norm_rect_in_for_hand_detector >> hand_detector.In("NORM_RECT");
-    auto hand_rects_from_hand_detector = hand_detector.Out("HAND_RECTS");
-
-    auto& hand_association = graph.AddNode("HandAssociationCalculator");
-    hand_association.GetOptions<HandAssociationCalculatorOptions>()
-        .set_min_similarity_threshold(tasks_options.min_tracking_confidence());
-    prev_hand_rects_from_landmarks >>
-        hand_association[Input<std::vector<NormalizedRect>>::Multiple("")][0];
-    hand_rects_from_hand_detector >>
-        hand_association[Input<std::vector<NormalizedRect>>::Multiple("")][1];
-    auto hand_rects = hand_association.Out("");
-
     auto& clip_hand_rects =
         graph.AddNode("ClipNormalizedRectVectorSizeCalculator");
     clip_hand_rects.GetOptions<ClipVectorSizeCalculatorOptions>()
         .set_max_vec_size(max_num_hands);
-    hand_rects >> clip_hand_rects.In("");
+
+    if (tasks_options.base_options().use_stream_mode()) {
+      // While in stream mode, skip hand detector graph when we successfully
+      // track the hands from the last frame.
+      auto image_for_hand_detector =
+          DisallowIf(image_in, has_enough_hands, graph);
+      auto norm_rect_in_for_hand_detector =
+          DisallowIf(norm_rect_in, has_enough_hands, graph);
+      image_for_hand_detector >> hand_detector.In("IMAGE");
+      norm_rect_in_for_hand_detector >> hand_detector.In("NORM_RECT");
+      auto hand_rects_from_hand_detector = hand_detector.Out("HAND_RECTS");
+      auto& hand_association = graph.AddNode("HandAssociationCalculator");
+      hand_association.GetOptions<HandAssociationCalculatorOptions>()
+          .set_min_similarity_threshold(
+              tasks_options.min_tracking_confidence());
+      prev_hand_rects_from_landmarks >>
+          hand_association[Input<std::vector<NormalizedRect>>::Multiple("")][0];
+      hand_rects_from_hand_detector >>
+          hand_association[Input<std::vector<NormalizedRect>>::Multiple("")][1];
+      auto hand_rects = hand_association.Out("");
+      hand_rects >> clip_hand_rects.In("");
+    } else {
+      // While not in stream mode, the input images are not guaranteed to be in
+      // series, and we don't want to enable the tracking and hand associations
+      // between input images. Always use the hand detector graph.
+      image_in >> hand_detector.In("IMAGE");
+      norm_rect_in >> hand_detector.In("NORM_RECT");
+      auto hand_rects_from_hand_detector = hand_detector.Out("HAND_RECTS");
+      hand_rects_from_hand_detector >> clip_hand_rects.In("");
+    }
     auto clipped_hand_rects = clip_hand_rects.Out("");
 
     auto& hand_landmarks_detector_graph = graph.AddNode(

mediapipe/tasks/java/com/google/mediapipe/tasks/audio/BUILD

@@ -0,0 +1,84 @@
+# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+android_library(
+    name = "core",
+    srcs = glob(["core/*.java"]),
+    javacopts = [
+        "-Xep:AndroidJdkLibsChecker:OFF",
+    ],
+    deps = [
+        ":libmediapipe_tasks_audio_jni_lib",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework_no_mff",
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:audiodata",
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
+# The native library of all MediaPipe audio tasks.
+cc_binary(
+    name = "libmediapipe_tasks_audio_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+        "//mediapipe/tasks/cc/audio/audio_classifier:audio_classifier_graph",
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni:model_resources_cache_jni",
+    ],
+)
+
+cc_library(
+    name = "libmediapipe_tasks_audio_jni_lib",
+    srcs = [":libmediapipe_tasks_audio_jni.so"],
+    alwayslink = 1,
+)
+
+android_library(
+    name = "audioclassifier",
+    srcs = [
+        "audioclassifier/AudioClassifier.java",
+        "audioclassifier/AudioClassifierResult.java",
+    ],
+    javacopts = [
+        "-Xep:AndroidJdkLibsChecker:OFF",
+    ],
+    manifest = "audioclassifier/AndroidManifest.xml",
+    deps = [
+        ":core",
+        "//mediapipe/framework:calculator_options_java_proto_lite",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/tasks/cc/audio/audio_classifier/proto:audio_classifier_graph_options_java_proto_lite",
+        "//mediapipe/tasks/cc/components/containers/proto:classifications_java_proto_lite",
+        "//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite",
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:audiodata",
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:classificationresult",
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors:classifieroptions",
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
+        "//third_party:autovalue",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
+load("//mediapipe/tasks/java/com/google/mediapipe/tasks:mediapipe_tasks_aar.bzl", "mediapipe_tasks_audio_aar")
+
+mediapipe_tasks_audio_aar(
+    name = "tasks_audio",
+    srcs = glob(["**/*.java"]),
+    native_library = ":libmediapipe_tasks_audio_jni_lib",
+)

mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioclassifier/AndroidManifest.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.tasks.audio.audioclassifier">
+
+    <uses-sdk android:minSdkVersion="24"
+        android:targetSdkVersion="30" />
+
+</manifest>

mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioclassifier/AudioClassifier.java

@@ -0,0 +1,399 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.audio.audioclassifier;
+
+import android.content.Context;
+import android.os.ParcelFileDescriptor;
+import com.google.auto.value.AutoValue;
+import com.google.mediapipe.proto.CalculatorOptionsProto.CalculatorOptions;
+import com.google.mediapipe.framework.MediaPipeException;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.framework.PacketGetter;
+import com.google.mediapipe.framework.ProtoUtil;
+import com.google.mediapipe.tasks.audio.audioclassifier.proto.AudioClassifierGraphOptionsProto;
+import com.google.mediapipe.tasks.audio.core.BaseAudioTaskApi;
+import com.google.mediapipe.tasks.audio.core.RunningMode;
+import com.google.mediapipe.tasks.components.containers.AudioData;
+import com.google.mediapipe.tasks.components.containers.proto.ClassificationsProto;
+import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
+import com.google.mediapipe.tasks.core.BaseOptions;
+import com.google.mediapipe.tasks.core.ErrorListener;
+import com.google.mediapipe.tasks.core.OutputHandler;
+import com.google.mediapipe.tasks.core.OutputHandler.PureResultListener;
+import com.google.mediapipe.tasks.core.OutputHandler.ResultListener;
+import com.google.mediapipe.tasks.core.TaskInfo;
+import com.google.mediapipe.tasks.core.TaskOptions;
+import com.google.mediapipe.tasks.core.TaskRunner;
+import com.google.mediapipe.tasks.core.proto.BaseOptionsProto;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Performs audio classification on audio clips or audio stream.
+ *
+ * <p>This API expects a TFLite model with mandatory TFLite Model Metadata that contains the
+ * mandatory AudioProperties of the solo input audio tensor and the optional (but recommended) label
+ * items as AssociatedFiles with type TENSOR_AXIS_LABELS per output classification tensor.
+ *
+ * <p>Input tensor: (kTfLiteFloat32)
+ *
+ * <ul>
+ *   <li>input audio buffer of size `[batch * samples]`.
+ *   <li>batch inference is not supported (`batch` is required to be 1).
+ *   <li>for multi-channel models, the channels need be interleaved.
+ * </ul>
+ *
+ * <p>At least one output tensor with: (kTfLiteFloat32)
+ *
+ * <ul>
+ *   <li>`[1 x N]` array with `N` represents the number of categories.
+ *   <li>optional (but recommended) label items as AssociatedFiles with type TENSOR_AXIS_LABELS,
+ *       containing one label per line. The first such AssociatedFile (if any) is used to fill the
+ *       `category_name` field of the results. The `display_name` field is filled from the
+ *       AssociatedFile (if any) whose locale matches the `display_names_locale` field of the
+ *       `AudioClassifierOptions` used at creation time ("en" by default, i.e. English). If none of
+ *       these are available, only the `index` field of the results will be filled.
+ * </ul>
+ */
+public final class AudioClassifier extends BaseAudioTaskApi {
+  private static final String TAG = AudioClassifier.class.getSimpleName();
+  private static final String AUDIO_IN_STREAM_NAME = "audio_in";
+  private static final String SAMPLE_RATE_IN_STREAM_NAME = "sample_rate_in";
+  private static final List<String> INPUT_STREAMS =
+      Collections.unmodifiableList(
+          Arrays.asList(
+              "AUDIO:" + AUDIO_IN_STREAM_NAME, "SAMPLE_RATE:" + SAMPLE_RATE_IN_STREAM_NAME));
+  private static final List<String> OUTPUT_STREAMS =
+      Collections.unmodifiableList(
+          Arrays.asList(
+              "CLASSIFICATIONS:classifications_out",
+              "TIMESTAMPED_CLASSIFICATIONS:timestamped_classifications_out"));
+  private static final int CLASSIFICATIONS_OUT_STREAM_INDEX = 0;
+  private static final int TIMESTAMPED_CLASSIFICATIONS_OUT_STREAM_INDEX = 1;
+  private static final String TASK_GRAPH_NAME =
+      "mediapipe.tasks.audio.audio_classifier.AudioClassifierGraph";
+  private static final long MICROSECONDS_PER_MILLISECOND = 1000;
+
+  static {
+    ProtoUtil.registerTypeName(
+        ClassificationsProto.ClassificationResult.class,
+        "mediapipe.tasks.components.containers.proto.ClassificationResult");
+  }
+
+  /**
+   * Creates an {@link AudioClassifier} instance from a model file and default {@link
+   * AudioClassifierOptions}.
+   *
+   * @param context an Android {@link Context}.
+   * @param modelPath path to the classification model in the assets.
+   * @throws MediaPipeException if there is an error during {@link AudioClassifier} creation.
+   */
+  public static AudioClassifier createFromFile(Context context, String modelPath) {
+    BaseOptions baseOptions = BaseOptions.builder().setModelAssetPath(modelPath).build();
+    return createFromOptions(
+        context, AudioClassifierOptions.builder().setBaseOptions(baseOptions).build());
+  }
+
+  /**
+   * Creates an {@link AudioClassifier} instance from a model file and default {@link
+   * AudioClassifierOptions}.
+   *
+   * @param context an Android {@link Context}.
+   * @param modelFile the classification model {@link File} instance.
+   * @throws IOException if an I/O error occurs when opening the tflite model file.
+   * @throws MediaPipeException if there is an error during {@link AudioClassifier} creation.
+   */
+  public static AudioClassifier createFromFile(Context context, File modelFile) throws IOException {
+    try (ParcelFileDescriptor descriptor =
+        ParcelFileDescriptor.open(modelFile, ParcelFileDescriptor.MODE_READ_ONLY)) {
+      BaseOptions baseOptions =
+          BaseOptions.builder().setModelAssetFileDescriptor(descriptor.getFd()).build();
+      return createFromOptions(
+          context, AudioClassifierOptions.builder().setBaseOptions(baseOptions).build());
+    }
+  }
+
+  /**
+   * Creates an {@link AudioClassifier} instance from a model buffer and default {@link
+   * AudioClassifierOptions}.
+   *
+   * @param context an Android {@link Context}.
+   * @param modelBuffer a direct {@link ByteBuffer} or a {@link MappedByteBuffer} of the
+   *     classification model.
+   * @throws MediaPipeException if there is an error during {@link AudioClassifier} creation.
+   */
+  public static AudioClassifier createFromBuffer(Context context, final ByteBuffer modelBuffer) {
+    BaseOptions baseOptions = BaseOptions.builder().setModelAssetBuffer(modelBuffer).build();
+    return createFromOptions(
+        context, AudioClassifierOptions.builder().setBaseOptions(baseOptions).build());
+  }
+
+  /**
+   * Creates an {@link AudioClassifier} instance from an {@link AudioClassifierOptions} instance.
+   *
+   * @param context an Android {@link Context}.
+   * @param options an {@link AudioClassifierOptions} instance.
+   * @throws MediaPipeException if there is an error during {@link AudioClassifier} creation.
+   */
+  public static AudioClassifier createFromOptions(Context context, AudioClassifierOptions options) {
+    OutputHandler<AudioClassifierResult, Void> handler = new OutputHandler<>();
+    handler.setOutputPacketConverter(
+        new OutputHandler.OutputPacketConverter<AudioClassifierResult, Void>() {
+          @Override
+          public AudioClassifierResult convertToTaskResult(List<Packet> packets) {
+            try {
+              if (!packets.get(CLASSIFICATIONS_OUT_STREAM_INDEX).isEmpty()) {
+                // For audio stream mode.
+                return AudioClassifierResult.createFromProto(
+                    PacketGetter.getProto(
+                        packets.get(CLASSIFICATIONS_OUT_STREAM_INDEX),
+                        ClassificationsProto.ClassificationResult.getDefaultInstance()),
+                    packets.get(CLASSIFICATIONS_OUT_STREAM_INDEX).getTimestamp()
+                        / MICROSECONDS_PER_MILLISECOND);
+              } else {
+                // For audio clips mode.
+                return AudioClassifierResult.createFromProtoList(
+                    PacketGetter.getProtoVector(
+                        packets.get(TIMESTAMPED_CLASSIFICATIONS_OUT_STREAM_INDEX),
+                        ClassificationsProto.ClassificationResult.parser()),
+                    -1);
+              }
+            } catch (IOException e) {
+              throw new MediaPipeException(
+                  MediaPipeException.StatusCode.INTERNAL.ordinal(), e.getMessage());
+            }
+          }
+
+          @Override
+          public Void convertToTaskInput(List<Packet> packets) {
+            return null;
+          }
+        });
+    if (options.resultListener().isPresent()) {
+      ResultListener<AudioClassifierResult, Void> resultListener =
+          new ResultListener<AudioClassifierResult, Void>() {
+            @Override
+            public void run(AudioClassifierResult audioClassifierResult, Void input) {
+              options.resultListener().get().run(audioClassifierResult);
+            }
+          };
+      handler.setResultListener(resultListener);
+    }
+    options.errorListener().ifPresent(handler::setErrorListener);
+    // Audio tasks should not drop input audio due to flow limiting, which may cause data
+    // inconsistency.
+    TaskRunner runner =
+        TaskRunner.create(
+            context,
+            TaskInfo.<AudioClassifierOptions>builder()
+                .setTaskGraphName(TASK_GRAPH_NAME)
+                .setInputStreams(INPUT_STREAMS)
+                .setOutputStreams(OUTPUT_STREAMS)
+                .setTaskOptions(options)
+                .setEnableFlowLimiting(false)
+                .build(),
+            handler);
+    return new AudioClassifier(runner, options.runningMode());
+  }
+
+  /**
+   * Constructor to initialize an {@link AudioClassifier} from a {@link TaskRunner} and {@link
+   * RunningMode}.
+   *
+   * @param taskRunner a {@link TaskRunner}.
+   * @param runningMode a mediapipe audio task {@link RunningMode}.
+   */
+  private AudioClassifier(TaskRunner taskRunner, RunningMode runningMode) {
+    super(taskRunner, runningMode, AUDIO_IN_STREAM_NAME, SAMPLE_RATE_IN_STREAM_NAME);
+  }
+
+  /*
+   * Performs audio classification on the provided audio clip. Only use this method when the
+   * AudioClassifier is created with the audio clips mode.
+   *
+   * <p>The audio clip is represented as a MediaPipe {@link AudioData} object  The method accepts
+   * audio clips with various length and audio sample rate.  It's required to provide the
+   * corresponding audio sample rate within the {@link AudioData} object.
+   *
+   * <p>The input audio clip may be longer than what the model is able to process in a single
+   * inference. When this occurs, the input audio clip is split into multiple chunks starting at
+   * different timestamps. For this reason, this function returns a vector of ClassificationResult
+   * objects, each associated with a timestamp corresponding to the start (in milliseconds) of the
+   * chunk data that was classified, e.g:
+   *
+   * ClassificationResult #0 (first chunk of data):
+   *  timestamp_ms: 0 (starts at 0ms)
+   *  classifications #0 (single head model):
+   *   category #0:
+   *    category_name: "Speech"
+   *    score: 0.6
+   *   category #1:
+   *    category_name: "Music"
+   *    score: 0.2
+   * ClassificationResult #1 (second chunk of data):
+   *  timestamp_ms: 800 (starts at 800ms)
+   *  classifications #0 (single head model):
+   *   category #0:
+   *    category_name: "Speech"
+   *    score: 0.5
+   *   category #1:
+   *    category_name: "Silence"
+   *    score: 0.1
+   *
+   * @param audioClip a MediaPipe {@link AudioData} object for processing.
+   * @throws MediaPipeException if there is an internal error.
+   */
+  public AudioClassifierResult classify(AudioData audioClip) {
+    return (AudioClassifierResult) processAudioClip(audioClip);
+  }
+
+  /*
+   * Sends audio data (a block in a continuous audio stream) to perform audio classification. Only
+   * use this method when the AudioClassifier is created with the audio stream mode.
+   *
+   * <p>The audio block is represented as a MediaPipe {@link AudioData} object. The audio data will
+   * be resampled, accumulated, and framed to the proper size for the underlying model to consume.
+   * It's required to provide the corresponding audio sample rate within {@link AudioData} object as
+   * well as a timestamp (in milliseconds) to indicate the start time of the input audio block. The
+   * timestamps must be monotonically increasing. This method will return immediately after
+   * the input audio data is accepted. The results will be available in the `resultListener`
+   * provided in the `AudioClassifierOptions`. The `classifyAsync` method is designed to process
+   * auido stream data such as microphone input.
+   *
+   * <p>The input audio block may be longer than what the model is able to process in a single
+   * inference. When this occurs, the input audio block is split into multiple chunks. For this
+   * reason, the callback may be called multiple times (once per chunk) for each call to this
+   * function.
+   *
+   * @param audioBlock a MediaPipe {@link AudioData} object for processing.
+   * @param timestampMs the input timestamp (in milliseconds).
+   * @throws MediaPipeException if there is an internal error.
+   */
+  public void classifyAsync(AudioData audioBlock, long timestampMs) {
+    checkOrSetSampleRate(audioBlock.getFormat().getSampleRate());
+    sendAudioStreamData(audioBlock, timestampMs);
+  }
+
+  /** Options for setting up and {@link AudioClassifier}. */
+  @AutoValue
+  public abstract static class AudioClassifierOptions extends TaskOptions {
+
+    /** Builder for {@link AudioClassifierOptions}. */
+    @AutoValue.Builder
+    public abstract static class Builder {
+      /** Sets the {@link BaseOptions} for the audio classifier task. */
+      public abstract Builder setBaseOptions(BaseOptions baseOptions);
+
+      /**
+       * Sets the {@link RunningMode} for the audio classifier task. Default to the audio clips
+       * mode. Image classifier has two modes:
+       *
+       * <ul>
+       *   <li>AUDIO_CLIPS: The mode for running audio classification on audio clips. Users feed
+       *       audio clips to the `classify` method, and will receive the classification results as
+       *       the return value.
+       *   <li>AUDIO_STREAM: The mode for running audio classification on the audio stream, such as
+       *       from microphone. Users call `classifyAsync` to push the audio data into the
+       *       AudioClassifier, the classification results will be available in the result callback
+       *       when the audio classifier finishes the work.
+       * </ul>
+       */
+      public abstract Builder setRunningMode(RunningMode runningMode);
+
+      /**
+       * Sets the optional {@link ClassifierOptions} controling classification behavior, such as
+       * score threshold, number of results, etc.
+       */
+      public abstract Builder setClassifierOptions(ClassifierOptions classifierOptions);
+
+      /**
+       * Sets the {@link ResultListener} to receive the classification results asynchronously when
+       * the audio classifier is in the audio stream mode.
+       */
+      public abstract Builder setResultListener(
+          PureResultListener<AudioClassifierResult> resultListener);
+
+      /** Sets an optional {@link ErrorListener}. */
+      public abstract Builder setErrorListener(ErrorListener errorListener);
+
+      abstract AudioClassifierOptions autoBuild();
+
+      /**
+       * Validates and builds the {@link AudioClassifierOptions} instance.
+       *
+       * @throws IllegalArgumentException if the result listener and the running mode are not
+       *     properly configured. The result listener should only be set when the audio classifier
+       *     is in the audio stream mode.
+       */
+      public final AudioClassifierOptions build() {
+        AudioClassifierOptions options = autoBuild();
+        if (options.runningMode() == RunningMode.AUDIO_STREAM) {
+          if (!options.resultListener().isPresent()) {
+            throw new IllegalArgumentException(
+                "The audio classifier is in the audio stream mode, a user-defined result listener"
+                    + " must be provided in the AudioClassifierOptions.");
+          }
+        } else if (options.resultListener().isPresent()) {
+          throw new IllegalArgumentException(
+              "The audio classifier is in the audio clips mode, a user-defined result listener"
+                  + " shouldn't be provided in AudioClassifierOptions.");
+        }
+        return options;
+      }
+    }
+
+    abstract BaseOptions baseOptions();
+
+    abstract RunningMode runningMode();
+
+    abstract Optional<ClassifierOptions> classifierOptions();
+
+    abstract Optional<PureResultListener<AudioClassifierResult>> resultListener();
+
+    abstract Optional<ErrorListener> errorListener();
+
+    public static Builder builder() {
+      return new AutoValue_AudioClassifier_AudioClassifierOptions.Builder()
+          .setRunningMode(RunningMode.AUDIO_CLIPS);
+    }
+
+    /**
+     * Converts a {@link AudioClassifierOptions} to a {@link CalculatorOptions} protobuf message.
+     */
+    @Override
+    public CalculatorOptions convertToCalculatorOptionsProto() {
+      BaseOptionsProto.BaseOptions.Builder baseOptionsBuilder =
+          BaseOptionsProto.BaseOptions.newBuilder();
+      baseOptionsBuilder.setUseStreamMode(runningMode() == RunningMode.AUDIO_STREAM);
+      baseOptionsBuilder.mergeFrom(convertBaseOptionsToProto(baseOptions()));
+      AudioClassifierGraphOptionsProto.AudioClassifierGraphOptions.Builder taskOptionsBuilder =
+          AudioClassifierGraphOptionsProto.AudioClassifierGraphOptions.newBuilder()
+              .setBaseOptions(baseOptionsBuilder);
+      if (classifierOptions().isPresent()) {
+        taskOptionsBuilder.setClassifierOptions(classifierOptions().get().convertToProto());
+      }
+      return CalculatorOptions.newBuilder()
+          .setExtension(
+              AudioClassifierGraphOptionsProto.AudioClassifierGraphOptions.ext,
+              taskOptionsBuilder.build())
+          .build();
+    }
+  }
+}

mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioclassifier/AudioClassifierResult.java

@@ -0,0 +1,76 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.audio.audioclassifier;
+
+import com.google.auto.value.AutoValue;
+import com.google.mediapipe.tasks.components.containers.ClassificationResult;
+import com.google.mediapipe.tasks.components.containers.proto.ClassificationsProto;
+import com.google.mediapipe.tasks.core.TaskResult;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/** Represents the classification results generated by {@link AudioClassifier}. */
+@AutoValue
+public abstract class AudioClassifierResult implements TaskResult {
+
+  /**
+   * Creates an {@link AudioClassifierResult} instance from a list of {@link
+   * ClassificationsProto.ClassificationResult} protobuf messages.
+   *
+   * @param protoList a list of {@link ClassificationsProto.ClassificationResult} protobuf message
+   *     to convert.
+   * @param timestampMs a timestamp for this result.
+   */
+  static AudioClassifierResult createFromProtoList(
+      List<ClassificationsProto.ClassificationResult> protoList, long timestampMs) {
+    List<ClassificationResult> classificationResultList = new ArrayList<>();
+    for (ClassificationsProto.ClassificationResult proto : protoList) {
+      classificationResultList.add(ClassificationResult.createFromProto(proto));
+    }
+    return new AutoValue_AudioClassifierResult(
+        Optional.of(classificationResultList), Optional.empty(), timestampMs);
+  }
+
+  /**
+   * Creates an {@link AudioClassifierResult} instance from a {@link
+   * ClassificationsProto.ClassificationResult} protobuf message.
+   *
+   * @param proto the {@link ClassificationsProto.ClassificationResult} protobuf message to convert.
+   * @param timestampMs a timestamp for this result.
+   */
+  static AudioClassifierResult createFromProto(
+      ClassificationsProto.ClassificationResult proto, long timestampMs) {
+    return new AutoValue_AudioClassifierResult(
+        Optional.empty(), Optional.of(ClassificationResult.createFromProto(proto)), timestampMs);
+  }
+
+  /**
+   * A list of of timpstamed {@link ClassificationResult} objects, each contains one set of results
+   * per classifier head. The list represents the audio classification result of an audio clip, and
+   * is only available when running with the audio clips mode.
+   */
+  public abstract Optional<List<ClassificationResult>> classificationResultList();
+
+  /**
+   * Contains one set of results per classifier head. A {@link ClassificationResult} usually
+   * represents one audio classification result in an audio stream, and s only available when
+   * running with the audio stream mode.
+   */
+  public abstract Optional<ClassificationResult> classificationResult();
+
+  @Override
+  public abstract long timestampMs();
+}

mediapipe/tasks/java/com/google/mediapipe/tasks/audio/core/BaseAudioTaskApi.java

@@ -0,0 +1,151 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.audio.core;
+
+import com.google.mediapipe.framework.MediaPipeException;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.tasks.components.containers.AudioData;
+import com.google.mediapipe.tasks.core.TaskResult;
+import com.google.mediapipe.tasks.core.TaskRunner;
+import java.util.HashMap;
+import java.util.Map;
+
+/** The base class of MediaPipe audio tasks. */
+public class BaseAudioTaskApi implements AutoCloseable {
+  private static final long MICROSECONDS_PER_MILLISECOND = 1000;
+  private static final long PRESTREAM_TIMESTAMP = Long.MIN_VALUE + 2;
+
+  private final TaskRunner runner;
+  private final RunningMode runningMode;
+  private final String audioStreamName;
+  private final String sampleRateStreamName;
+  private double defaultSampleRate;
+
+  static {
+    System.loadLibrary("mediapipe_tasks_audio_jni");
+  }
+
+  /**
+   * Constructor to initialize a {@link BaseAudioTaskApi}.
+   *
+   * @param runner a {@link TaskRunner}.
+   * @param runningMode a mediapipe audio task {@link RunningMode}.
+   * @param audioStreamName the name of the input audio stream.
+   * @param sampleRateStreamName the name of the audio sample rate stream.
+   */
+  public BaseAudioTaskApi(
+      TaskRunner runner,
+      RunningMode runningMode,
+      String audioStreamName,
+      String sampleRateStreamName) {
+    this.runner = runner;
+    this.runningMode = runningMode;
+    this.audioStreamName = audioStreamName;
+    this.sampleRateStreamName = sampleRateStreamName;
+    this.defaultSampleRate = -1.0;
+  }
+
+  /**
+   * A synchronous method to process audio clips. The call blocks the current thread until a failure
+   * status or a successful result is returned.
+   *
+   * @param audioClip a MediaPipe {@link AudioDatra} object for processing.
+   * @throws MediaPipeException if the task is not in the audio clips mode.
+   */
+  protected TaskResult processAudioClip(AudioData audioClip) {
+    if (runningMode != RunningMode.AUDIO_CLIPS) {
+      throw new MediaPipeException(
+          MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+          "Task is not initialized with the audio clips mode. Current running mode:"
+              + runningMode.name());
+    }
+    Map<String, Packet> inputPackets = new HashMap<>();
+    inputPackets.put(
+        audioStreamName,
+        runner
+            .getPacketCreator()
+            .createMatrix(
+                audioClip.getFormat().getNumOfChannels(),
+                audioClip.getBufferLength(),
+                audioClip.getBuffer()));
+    inputPackets.put(
+        sampleRateStreamName,
+        runner.getPacketCreator().createFloat64(audioClip.getFormat().getSampleRate()));
+    return runner.process(inputPackets);
+  }
+
+  /**
+   * Checks or sets the audio sample rate in the audio stream mode.
+   *
+   * @param sampleRate the audio sample rate.
+   * @throws MediaPipeException if the task is not in the audio stream mode or the provided sample
+   *     rate is inconsisent with the previously recevied.
+   */
+  protected void checkOrSetSampleRate(double sampleRate) {
+    if (runningMode != RunningMode.AUDIO_STREAM) {
+      throw new MediaPipeException(
+          MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+          "Task is not initialized with the audio stream mode. Current running mode:"
+              + runningMode.name());
+    }
+    if (defaultSampleRate > 0) {
+      if (Double.compare(sampleRate, defaultSampleRate) != 0) {
+        throw new MediaPipeException(
+            MediaPipeException.StatusCode.INVALID_ARGUMENT.ordinal(),
+            "The input audio sample rate: "
+                + sampleRate
+                + " is inconsistent with the previously provided: "
+                + defaultSampleRate);
+      }
+    } else {
+      Map<String, Packet> inputPackets = new HashMap<>();
+      inputPackets.put(sampleRateStreamName, runner.getPacketCreator().createFloat64(sampleRate));
+      runner.send(inputPackets, PRESTREAM_TIMESTAMP);
+      defaultSampleRate = sampleRate;
+    }
+  }
+  /**
+   * An asynchronous method to send audio stream data to the {@link TaskRunner}. The results will be
+   * available in the user-defined result listener.
+   *
+   * @param audioClip a MediaPipe {@link AudioDatra} object for processing.
+   * @param timestampMs the corresponding timestamp of the input image in milliseconds.
+   * @throws MediaPipeException if the task is not in the stream mode.
+   */
+  protected void sendAudioStreamData(AudioData audioClip, long timestampMs) {
+    if (runningMode != RunningMode.AUDIO_STREAM) {
+      throw new MediaPipeException(
+          MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+          "Task is not initialized with the audio stream mode. Current running mode:"
+              + runningMode.name());
+    }
+    Map<String, Packet> inputPackets = new HashMap<>();
+    inputPackets.put(
+        audioStreamName,
+        runner
+            .getPacketCreator()
+            .createMatrix(
+                audioClip.getFormat().getNumOfChannels(),
+                audioClip.getBufferLength(),
+                audioClip.getBuffer()));
+    runner.send(inputPackets, timestampMs * MICROSECONDS_PER_MILLISECOND);
+  }
+
+  /** Closes and cleans up the MediaPipe audio task. */
+  @Override
+  public void close() {
+    runner.close();
+  }
+}

mediapipe/tasks/java/com/google/mediapipe/tasks/audio/core/RunningMode.java

@@ -0,0 +1,29 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.audio.core;
+
+/**
+ * MediaPipe audio task running mode. A MediaPipe audio task can be run with two different modes:
+ *
+ * <ul>
+ *   <li>AUDIO_CLIPS: The mode for running a mediapipe audio task on independent audio clips.
+ *   <li>AUDIO_STREAM: The mode for running a mediapipe audio task on an audio stream, such as from
+ *       microphone.
+ * </ul>
+ */
+public enum RunningMode {
+  AUDIO_CLIPS,
+  AUDIO_STREAM,
+}

mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/AudioData.java

@@ -0,0 +1,334 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.components.containers;
+
+import static java.lang.System.arraycopy;
+
+import android.media.AudioFormat;
+import android.media.AudioRecord;
+import com.google.auto.value.AutoValue;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.FloatBuffer;
+
+/**
+ * Defines a ring buffer and some utility functions to prepare the input audio samples.
+ *
+ * <p>It maintains a <a href="https://en.wikipedia.org/wiki/Circular_buffer">Ring Buffer</a> to hold
+ * input audio data. Clients could feed input audio data via `load` methods and access the
+ * aggregated audio samples via `getTensorBuffer` method.
+ *
+ * <p>Note that this class can only handle input audio in Float (in {@link
+ * android.media.AudioFormat#ENCODING_PCM_16BIT}) or Short (in {@link
+ * android.media.AudioFormat#ENCODING_PCM_FLOAT}). Internally it converts and stores all the audio
+ * samples in PCM Float encoding.
+ *
+ * <p>Typical usage in Kotlin
+ *
+ * <pre>
+ *   val audioData = AudioData.create(format, modelInputLength)
+ *   audioData.load(newData)
+ * </pre>
+ *
+ * <p>Another sample usage with {@link android.media.AudioRecord}
+ *
+ * <pre>
+ *   val audioData = AudioData.create(format, modelInputLength)
+ *   Timer().scheduleAtFixedRate(delay, period) {
+ *     audioData.load(audioRecord)
+ *   }
+ * </pre>
+ */
+public class AudioData {
+
+  private static final String TAG = AudioData.class.getSimpleName();
+  private final FloatRingBuffer buffer;
+  private final AudioDataFormat format;
+
+  /**
+   * Creates a {@link android.media.AudioRecord} instance with a ring buffer whose size is {@code
+   * sampleCounts} * {@code format.getNumOfChannels()}.
+   *
+   * @param format the expected {@link AudioDataFormat} of audio data loaded into this class.
+   * @param sampleCounts the number of samples.
+   */
+  public static AudioData create(AudioDataFormat format, int sampleCounts) {
+    return new AudioData(format, sampleCounts);
+  }
+
+  /**
+   * Creates a {@link AudioData} instance with a ring buffer whose size is {@code sampleCounts} *
+   * {@code format.getChannelCount()}.
+   *
+   * @param format the {@link android.media.AudioFormat} required by the TFLite model. It defines
+   *     the number of channels and sample rate.
+   * @param sampleCounts the number of samples to be fed into the model
+   */
+  public static AudioData create(AudioFormat format, int sampleCounts) {
+    return new AudioData(AudioDataFormat.create(format), sampleCounts);
+  }
+
+  /**
+   * Wraps a few constants describing the format of the incoming audio samples, namely number of
+   * channels and the sample rate. By default, num of channels is set to 1.
+   */
+  @AutoValue
+  public abstract static class AudioDataFormat {
+    private static final int DEFAULT_NUM_OF_CHANNELS = 1;
+
+    /** Creates a {@link AudioFormat} instance from Android AudioFormat class. */
+    public static AudioDataFormat create(AudioFormat format) {
+      return AudioDataFormat.builder()
+          .setNumOfChannels(format.getChannelCount())
+          .setSampleRate(format.getSampleRate())
+          .build();
+    }
+
+    public abstract int getNumOfChannels();
+
+    public abstract float getSampleRate();
+
+    public static Builder builder() {
+      return new AutoValue_AudioData_AudioDataFormat.Builder()
+          .setNumOfChannels(DEFAULT_NUM_OF_CHANNELS);
+    }
+
+    /** Builder for {@link AudioDataFormat} */
+    @AutoValue.Builder
+    public abstract static class Builder {
+
+      /* By default, it's set to have 1 channel. */
+      public abstract Builder setNumOfChannels(int value);
+
+      public abstract Builder setSampleRate(float value);
+
+      abstract AudioDataFormat autoBuild();
+
+      public AudioDataFormat build() {
+        AudioDataFormat format = autoBuild();
+        if (format.getNumOfChannels() <= 0) {
+          throw new IllegalArgumentException("Number of channels should be greater than 0");
+        }
+        if (format.getSampleRate() <= 0) {
+          throw new IllegalArgumentException("Sample rate should be greater than 0");
+        }
+        return format;
+      }
+    }
+  }
+
+  /**
+   * Stores the input audio samples {@code src} in the ring buffer.
+   *
+   * @param src input audio samples in {@link android.media.AudioFormat#ENCODING_PCM_FLOAT}. For
+   *     multi-channel input, the array is interleaved.
+   */
+  public void load(float[] src) {
+    load(src, 0, src.length);
+  }
+
+  /**
+   * Stores the input audio samples {@code src} in the ring buffer.
+   *
+   * @param src input audio samples in {@link android.media.AudioFormat#ENCODING_PCM_FLOAT}. For
+   *     multi-channel input, the array is interleaved.
+   * @param offsetInFloat starting position in the {@code src} array
+   * @param sizeInFloat the number of float values to be copied
+   * @throws IllegalArgumentException for incompatible audio format or incorrect input size
+   */
+  public void load(float[] src, int offsetInFloat, int sizeInFloat) {
+    if (sizeInFloat % format.getNumOfChannels() != 0) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Size (%d) needs to be a multiplier of the number of channels (%d)",
+              sizeInFloat, format.getNumOfChannels()));
+    }
+    buffer.load(src, offsetInFloat, sizeInFloat);
+  }
+
+  /**
+   * Converts the input audio samples {@code src} to ENCODING_PCM_FLOAT, then stores it in the ring
+   * buffer.
+   *
+   * @param src input audio samples in {@link android.media.AudioFormat#ENCODING_PCM_16BIT}. For
+   *     multi-channel input, the array is interleaved.
+   */
+  public void load(short[] src) {
+    load(src, 0, src.length);
+  }
+
+  /**
+   * Converts the input audio samples {@code src} to ENCODING_PCM_FLOAT, then stores it in the ring
+   * buffer.
+   *
+   * @param src input audio samples in {@link android.media.AudioFormat#ENCODING_PCM_16BIT}. For
+   *     multi-channel input, the array is interleaved.
+   * @param offsetInShort starting position in the src array
+   * @param sizeInShort the number of short values to be copied
+   * @throws IllegalArgumentException if the source array can't be copied
+   */
+  public void load(short[] src, int offsetInShort, int sizeInShort) {
+    if (offsetInShort + sizeInShort > src.length) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Index out of range. offset (%d) + size (%d) should <= newData.length (%d)",
+              offsetInShort, sizeInShort, src.length));
+    }
+    float[] floatData = new float[sizeInShort];
+    for (int i = 0; i < sizeInShort; i++) {
+      // Convert the data to PCM Float encoding i.e. values between -1 and 1
+      floatData[i] = src[i + offsetInShort] * 1.f / Short.MAX_VALUE;
+    }
+    load(floatData);
+  }
+
+  /**
+   * Loads latest data from the {@link android.media.AudioRecord} in a non-blocking way. Only
+   * supporting ENCODING_PCM_16BIT and ENCODING_PCM_FLOAT.
+   *
+   * @param record an instance of {@link android.media.AudioRecord}
+   * @return number of captured audio values whose size is {@code channelCount * sampleCount}. If
+   *     there was no new data in the AudioRecord or an error occurred, this method will return 0.
+   * @throws IllegalArgumentException for unsupported audio encoding format
+   * @throws IllegalStateException if reading from AudioRecord failed
+   */
+  public int load(AudioRecord record) {
+    if (!this.format.equals(AudioDataFormat.create(record.getFormat()))) {
+      throw new IllegalArgumentException("Incompatible audio format.");
+    }
+    int loadedValues = 0;
+    if (record.getAudioFormat() == AudioFormat.ENCODING_PCM_FLOAT) {
+      float[] newData = new float[record.getChannelCount() * record.getBufferSizeInFrames()];
+      loadedValues = record.read(newData, 0, newData.length, AudioRecord.READ_NON_BLOCKING);
+      if (loadedValues > 0) {
+        load(newData, 0, loadedValues);
+        return loadedValues;
+      }
+    } else if (record.getAudioFormat() == AudioFormat.ENCODING_PCM_16BIT) {
+      short[] newData = new short[record.getChannelCount() * record.getBufferSizeInFrames()];
+      loadedValues = record.read(newData, 0, newData.length, AudioRecord.READ_NON_BLOCKING);
+      if (loadedValues > 0) {
+        load(newData, 0, loadedValues);
+        return loadedValues;
+      }
+    } else {
+      throw new IllegalArgumentException(
+          "Unsupported encoding. Requires ENCODING_PCM_16BIT or ENCODING_PCM_FLOAT.");
+    }
+
+    switch (loadedValues) {
+      case AudioRecord.ERROR_INVALID_OPERATION:
+        throw new IllegalStateException("AudioRecord.ERROR_INVALID_OPERATION");
+
+      case AudioRecord.ERROR_BAD_VALUE:
+        throw new IllegalStateException("AudioRecord.ERROR_BAD_VALUE");
+
+      case AudioRecord.ERROR_DEAD_OBJECT:
+        throw new IllegalStateException("AudioRecord.ERROR_DEAD_OBJECT");
+
+      case AudioRecord.ERROR:
+        throw new IllegalStateException("AudioRecord.ERROR");
+
+      default:
+        return 0;
+    }
+  }
+
+  /**
+   * Returns a float array holding all the available audio samples in {@link
+   * android.media.AudioFormat#ENCODING_PCM_FLOAT} i.e. values are in the range of [-1, 1].
+   */
+  public float[] getBuffer() {
+    float[] bufferData = new float[buffer.getCapacity()];
+    ByteBuffer byteBuffer = buffer.getBuffer();
+    byteBuffer.asFloatBuffer().get(bufferData);
+    return bufferData;
+  }
+
+  /* Returns the {@link AudioDataFormat} associated with the tensor. */
+  public AudioDataFormat getFormat() {
+    return format;
+  }
+
+  /* Returns the audio buffer length. */
+  public int getBufferLength() {
+    return buffer.getCapacity() / format.getNumOfChannels();
+  }
+
+  private AudioData(AudioDataFormat format, int sampleCounts) {
+    this.format = format;
+    this.buffer = new FloatRingBuffer(sampleCounts * format.getNumOfChannels());
+  }
+
+  /** Actual implementation of the ring buffer. */
+  private static class FloatRingBuffer {
+
+    private final float[] buffer;
+    private int nextIndex = 0;
+
+    public FloatRingBuffer(int flatSize) {
+      buffer = new float[flatSize];
+    }
+
+    /**
+     * Loads a slice of the float array to the ring buffer. If the float array is longer than ring
+     * buffer's capacity, samples with lower indices in the array will be ignored.
+     */
+    public void load(float[] newData, int offset, int size) {
+      if (offset + size > newData.length) {
+        throw new IllegalArgumentException(
+            String.format(
+                "Index out of range. offset (%d) + size (%d) should <= newData.length (%d)",
+                offset, size, newData.length));
+      }
+      // If buffer can't hold all the data, only keep the most recent data of size buffer.length
+      if (size > buffer.length) {
+        offset += (size - buffer.length);
+        size = buffer.length;
+      }
+      if (nextIndex + size < buffer.length) {
+        // No need to wrap nextIndex, just copy newData[offset:offset + size]
+        // to buffer[nextIndex:nextIndex+size]
+        arraycopy(newData, offset, buffer, nextIndex, size);
+      } else {
+        // Need to wrap nextIndex, perform copy in two chunks.
+        int firstChunkSize = buffer.length - nextIndex;
+        // First copy newData[offset:offset+firstChunkSize] to buffer[nextIndex:buffer.length]
+        arraycopy(newData, offset, buffer, nextIndex, firstChunkSize);
+        // Then copy newData[offset+firstChunkSize:offset+size] to buffer[0:size-firstChunkSize]
+        arraycopy(newData, offset + firstChunkSize, buffer, 0, size - firstChunkSize);
+      }
+
+      nextIndex = (nextIndex + size) % buffer.length;
+    }
+
+    public ByteBuffer getBuffer() {
+      // Create non-direct buffers. On Pixel 4, creating direct buffer costs around 0.1 ms, which
+      // can be 5x ~ 10x longer compared to non-direct buffer backed by arrays (around 0.01ms), so
+      // generally we don't create direct buffer for every invocation.
+      ByteBuffer byteBuffer = ByteBuffer.allocate(Float.SIZE / 8 * buffer.length);
+      byteBuffer.order(ByteOrder.nativeOrder());
+      FloatBuffer result = byteBuffer.asFloatBuffer();
+      result.put(buffer, nextIndex, buffer.length - nextIndex);
+      result.put(buffer, 0, nextIndex);
+      byteBuffer.rewind();
+      return byteBuffer;
+    }
+
+    public int getCapacity() {
+      return buffer.length;
+    }
+  }
+}

mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD

@@ -16,6 +16,15 @@ package(default_visibility = ["//mediapipe/tasks:internal"])
 
 licenses(["notice"])
 
+android_library(
+    name = "audiodata",
+    srcs = ["AudioData.java"],
+    deps = [
+        "//third_party:autovalue",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
 android_library(
     name = "category",
     srcs = ["Category.java"],
@@ -80,3 +89,30 @@ filegroup(
     srcs = glob(["*.java"]),
     visibility = ["//mediapipe/tasks/java/com/google/mediapipe/tasks/core:__subpackages__"],
 )
+
+android_library(
+    name = "embedding",
+    srcs = ["Embedding.java"],
+    javacopts = [
+        "-Xep:AndroidJdkLibsChecker:OFF",
+    ],
+    deps = [
+        "//mediapipe/tasks/cc/components/containers/proto:embeddings_java_proto_lite",
+        "//third_party:autovalue",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
+android_library(
+    name = "embeddingresult",
+    srcs = ["EmbeddingResult.java"],
+    javacopts = [
+        "-Xep:AndroidJdkLibsChecker:OFF",
+    ],
+    deps = [
+        ":embedding",
+        "//mediapipe/tasks/cc/components/containers/proto:embeddings_java_proto_lite",
+        "//third_party:autovalue",
+        "@maven//:com_google_guava_guava",
+    ],
+)

mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Embedding.java

@@ -0,0 +1,88 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.components.containers;
+
+import com.google.auto.value.AutoValue;
+import com.google.mediapipe.tasks.components.containers.proto.EmbeddingsProto;
+import java.util.Optional;
+
+/**
+ * Represents the embedding for a given embedder head. Typically used in embedding tasks.
+ *
+ * <p>One and only one of the two 'floatEmbedding' and 'quantizedEmbedding' will contain data, based
+ * on whether or not the embedder was configured to perform scala quantization.
+ */
+@AutoValue
+public abstract class Embedding {
+
+  /**
+   * Creates an {@link Embedding} instance.
+   *
+   * @param floatEmbedding the floating-point embedding
+   * @param quantizedEmbedding the quantized embedding.
+   * @param headIndex the index of the embedder head.
+   * @param headName the optional name of the embedder head.
+   */
+  public static Embedding create(
+      float[] floatEmbedding, byte[] quantizedEmbedding, int headIndex, Optional<String> headName) {
+    return new AutoValue_Embedding(floatEmbedding, quantizedEmbedding, headIndex, headName);
+  }
+
+  /**
+   * Creates an {@link Embedding} object from an {@link EmbeddingsProto.Embedding} protobuf message.
+   *
+   * @param proto the {@link EmbeddingsProto.Embedding} protobuf message to convert.
+   */
+  public static Embedding createFromProto(EmbeddingsProto.Embedding proto) {
+    float[] floatEmbedding;
+    if (proto.hasFloatEmbedding()) {
+      floatEmbedding = new float[proto.getFloatEmbedding().getValuesCount()];
+      for (int i = 0; i < floatEmbedding.length; i++) {
+        floatEmbedding[i] = proto.getFloatEmbedding().getValues(i);
+      }
+    } else {
+      floatEmbedding = new float[0];
+    }
+    return Embedding.create(
+        floatEmbedding,
+        proto.hasQuantizedEmbedding()
+            ? proto.getQuantizedEmbedding().getValues().toByteArray()
+            : new byte[0],
+        proto.getHeadIndex(),
+        proto.hasHeadName() ? Optional.of(proto.getHeadName()) : Optional.empty());
+  }
+
+  /**
+   * Floating-point embedding.
+   *
+   * <p>Empty if the embedder was configured to perform scalar quantization.
+   */
+  public abstract float[] floatEmbedding();
+
+  /**
+   * Quantized embedding.
+   *
+   * <p>Empty if the embedder was not configured to perform scalar quantization.
+   */
+  public abstract byte[] quantizedEmbedding();
+
+  /**
+   * The index of the embedder head these entries refer to. This is useful for multi-head models.
+   */
+  public abstract int headIndex();
+
+  /** The optional name of the embedder head, which is the corresponding tensor metadata name. */
+  public abstract Optional<String> headName();
+}

mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/EmbeddingResult.java

@@ -0,0 +1,69 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.components.containers;
+
+import com.google.auto.value.AutoValue;
+import com.google.mediapipe.tasks.components.containers.proto.EmbeddingsProto;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+/** Represents the embedding results of a model. Typically used as a result for embedding tasks. */
+@AutoValue
+public abstract class EmbeddingResult {
+
+  /**
+   * Creates a {@link EmbeddingResult} instance.
+   *
+   * @param embeddings the list of {@link Embedding} objects containing the embedding for each head
+   *     of the model.
+   * @param timestampMs the optional timestamp (in milliseconds) of the start of the chunk of data
+   *     corresponding to these results.
+   */
+  public static EmbeddingResult create(List<Embedding> embeddings, Optional<Long> timestampMs) {
+    return new AutoValue_EmbeddingResult(Collections.unmodifiableList(embeddings), timestampMs);
+  }
+
+  /**
+   * Creates a {@link EmbeddingResult} object from a {@link EmbeddingsProto.EmbeddingResult}
+   * protobuf message.
+   *
+   * @param proto the {@link EmbeddingsProto.EmbeddingResult} protobuf message to convert.
+   */
+  public static EmbeddingResult createFromProto(EmbeddingsProto.EmbeddingResult proto) {
+    List<Embedding> embeddings = new ArrayList<>();
+    for (EmbeddingsProto.Embedding embeddingProto : proto.getEmbeddingsList()) {
+      embeddings.add(Embedding.createFromProto(embeddingProto));
+    }
+    Optional<Long> timestampMs =
+        proto.hasTimestampMs() ? Optional.of(proto.getTimestampMs()) : Optional.empty();
+    return create(embeddings, timestampMs);
+  }
+
+  /** The embedding results for each head of the model. */
+  public abstract List<Embedding> embeddings();
+
+  /**
+   * The optional timestamp (in milliseconds) of the start of the chunk of data corresponding to
+   * these results.
+   *
+   * <p>This is only used for embedding extraction on time series (e.g. audio embedder). In these
+   * use cases, the amount of data to process might exceed the maximum size that the model can
+   * process: to solve this, the input data is split into multiple chunks starting at different
+   * timestamps.
+   */
+  public abstract Optional<Long> timestampMs();
+}

mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/BUILD

@@ -29,6 +29,19 @@ android_library(
     ],
 )
 
+android_library(
+    name = "embedderoptions",
+    srcs = ["EmbedderOptions.java"],
+    javacopts = [
+        "-Xep:AndroidJdkLibsChecker:OFF",
+    ],
+    deps = [
+        "//mediapipe/tasks/cc/components/processors/proto:embedder_options_java_proto_lite",
+        "//third_party:autovalue",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
 # Expose the java source files for building mediapipe tasks core AAR.
 filegroup(
     name = "java_src",

mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors/EmbedderOptions.java

@@ -0,0 +1,68 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.components.processors;
+
+import com.google.auto.value.AutoValue;
+import com.google.mediapipe.tasks.components.processors.proto.EmbedderOptionsProto;
+
+/** Embedder options shared across MediaPipe Java embedding tasks. */
+@AutoValue
+public abstract class EmbedderOptions {
+
+  /** Builder for {@link EmbedderOptions} */
+  @AutoValue.Builder
+  public abstract static class Builder {
+    /**
+     * Sets whether L2 normalization should be performed on the returned embeddings. Use this option
+     * only if the model does not already contain a native <code>L2_NORMALIZATION</code> TF Lite Op.
+     * In most cases, this is already the case and L2 norm is thus achieved through TF Lite
+     * inference.
+     *
+     * <p>False by default.
+     */
+    public abstract Builder setL2Normalize(boolean l2Normalize);
+
+    /**
+     * Sets whether the returned embedding should be quantized to bytes via scalar quantization.
+     * Embeddings are implicitly assumed to be unit-norm and therefore any dimensions is guaranteed
+     * to have value in <code>[-1.0, 1.0]</code>. Use {@link #setL2Normalize(boolean)} if this is
+     * not the case.
+     *
+     * <p>False by default.
+     */
+    public abstract Builder setQuantize(boolean quantize);
+
+    public abstract EmbedderOptions build();
+  }
+
+  public abstract boolean l2Normalize();
+
+  public abstract boolean quantize();
+
+  public static Builder builder() {
+    return new AutoValue_EmbedderOptions.Builder().setL2Normalize(false).setQuantize(false);
+  }
+
+  /**
+   * Converts an {@link EmbedderOptions} object to an {@link EmbedderOptionsProto.EmbedderOptions}
+   * protobuf message.
+   */
+  public EmbedderOptionsProto.EmbedderOptions convertToProto() {
+    return EmbedderOptionsProto.EmbedderOptions.newBuilder()
+        .setL2Normalize(l2Normalize())
+        .setQuantize(quantize())
+        .build();
+  }
+}

mediapipe/tasks/java/com/google/mediapipe/tasks/components/utils/BUILD

@@ -0,0 +1,33 @@
+# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+android_library(
+    name = "cosinesimilarity",
+    srcs = ["CosineSimilarity.java"],
+    deps = [
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:embedding",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
+# Expose the java source files for building mediapipe tasks core AAR.
+filegroup(
+    name = "java_src",
+    srcs = glob(["*.java"]),
+    visibility = ["//mediapipe/tasks/java/com/google/mediapipe/tasks/core:__subpackages__"],
+)

mediapipe/tasks/java/com/google/mediapipe/tasks/components/utils/CosineSimilarity.java

@@ -0,0 +1,88 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.components.utils;
+
+import com.google.mediapipe.tasks.components.containers.Embedding;
+
+/** Utility class for computing cosine similarity between {@link Embedding} objects. */
+public class CosineSimilarity {
+
+  // Non-instantiable class.
+  private CosineSimilarity() {}
+
+  /**
+   * Computes <a href="https://en.wikipedia.org/wiki/Cosine_similarity">cosine similarity</a>
+   * between two {@link Embedding} objects.
+   *
+   * @throws IllegalArgumentException if the embeddings are of different types (float vs.
+   *     quantized), have different sizes, or have an L2-norm of 0.
+   */
+  public static double compute(Embedding u, Embedding v) {
+    if (u.floatEmbedding().length > 0 && v.floatEmbedding().length > 0) {
+      return computeFloat(u.floatEmbedding(), v.floatEmbedding());
+    }
+    if (u.quantizedEmbedding().length > 0 && v.quantizedEmbedding().length > 0) {
+      return computeQuantized(u.quantizedEmbedding(), v.quantizedEmbedding());
+    }
+    throw new IllegalArgumentException(
+        "Cannot compute cosine similarity between quantized and float embeddings.");
+  }
+
+  private static double computeFloat(float[] u, float[] v) {
+    if (u.length != v.length) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Cannot compute cosine similarity between embeddings of different sizes (%d vs."
+                  + " %d).",
+              u.length, v.length));
+    }
+    double dotProduct = 0.0;
+    double normU = 0.0;
+    double normV = 0.0;
+    for (int i = 0; i < u.length; i++) {
+      dotProduct += u[i] * v[i];
+      normU += u[i] * u[i];
+      normV += v[i] * v[i];
+    }
+    if (normU <= 0 || normV <= 0) {
+      throw new IllegalArgumentException(
+          "Cannot compute cosine similarity on embedding with 0 norm.");
+    }
+    return dotProduct / Math.sqrt(normU * normV);
+  }
+
+  private static double computeQuantized(byte[] u, byte[] v) {
+    if (u.length != v.length) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Cannot compute cosine similarity between embeddings of different sizes (%d vs."
+                  + " %d).",
+              u.length, v.length));
+    }
+    double dotProduct = 0.0;
+    double normU = 0.0;
+    double normV = 0.0;
+    for (int i = 0; i < u.length; i++) {
+      dotProduct += u[i] * v[i];
+      normU += u[i] * u[i];
+      normV += v[i] * v[i];
+    }
+    if (normU <= 0 || normV <= 0) {
+      throw new IllegalArgumentException(
+          "Cannot compute cosine similarity on embedding with 0 norm.");
+    }
+    return dotProduct / Math.sqrt(normU * normV);
+  }
+}

mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD

@@ -44,6 +44,7 @@ mediapipe_tasks_core_aar(
     srcs = glob(["*.java"]) + [
         "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:java_src",
         "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors:java_src",
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/utils:java_src",
         "//mediapipe/java/com/google/mediapipe/framework/image:java_src",
     ],
     manifest = "AndroidManifest.xml",

mediapipe/tasks/java/com/google/mediapipe/tasks/core/OutputHandler.java

@@ -31,11 +31,22 @@ public class OutputHandler<OutputT extends TaskResult, InputT> {
     InputT convertToTaskInput(List<Packet> packets);
   }
 
-  /** Interface for the customizable MediaPipe task result listener. */
+  /**
+   * Interface for the customizable MediaPipe task result listener that can reteive both task result
+   * objects and the correpsonding input data.
+   */
   public interface ResultListener<OutputT extends TaskResult, InputT> {
     void run(OutputT result, InputT input);
   }
 
+  /**
+   * Interface for the customizable MediaPipe task result listener that can only reteive task result
+   * objects.
+   */
+  public interface PureResultListener<OutputT extends TaskResult> {
+    void run(OutputT result);
+  }
+
   private static final String TAG = "OutputHandler";
   // A task-specific graph output packet converter that should be implemented per task.
   private OutputPacketConverter<OutputT, InputT> outputPacketConverter;
@@ -45,6 +56,8 @@ public class OutputHandler<OutputT extends TaskResult, InputT> {
   protected ErrorListener errorListener;
   // The cached task result for non latency sensitive use cases.
   protected OutputT cachedTaskResult;
+  // The latest output timestamp.
+  protected long latestOutputTimestamp = -1;
   // Whether the output handler should react to timestamp-bound changes by outputting empty packets.
   private boolean handleTimestampBoundChanges = false;
 
@@ -98,6 +111,11 @@ public class OutputHandler<OutputT extends TaskResult, InputT> {
     return taskResult;
   }
 
+  /* Returns the latest output timestamp. */
+  public long getLatestOutputTimestamp() {
+    return latestOutputTimestamp;
+  }
+
   /**
    * Handles a list of output {@link Packet}s. Invoked when a packet list become available.
    *
@@ -109,6 +127,7 @@ public class OutputHandler<OutputT extends TaskResult, InputT> {
       taskResult = outputPacketConverter.convertToTaskResult(packets);
       if (resultListener == null) {
         cachedTaskResult = taskResult;
+        latestOutputTimestamp = packets.get(0).getTimestamp();
       } else {
         InputT taskInput = outputPacketConverter.convertToTaskInput(packets);
         resultListener.run(taskResult, taskInput);
@@ -119,12 +138,6 @@ public class OutputHandler<OutputT extends TaskResult, InputT> {
       } else {
         Log.e(TAG, "Error occurs when getting MediaPipe task result. " + e);
       }
-    } finally {
-      for (Packet packet : packets) {
-        if (packet != null) {
-          packet.release();
-        }
-      }
     }
   }
 }

mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskRunner.java

@@ -93,6 +93,7 @@ public class TaskRunner implements AutoCloseable {
   public synchronized TaskResult process(Map<String, Packet> inputs) {
     addPackets(inputs, generateSyntheticTimestamp());
     graph.waitUntilGraphIdle();
+    lastSeenTimestamp = outputHandler.getLatestOutputTimestamp();
     return outputHandler.retrieveCachedTaskResult();
   }
 

mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl

@@ -30,6 +30,10 @@ _CORE_TASKS_JAVA_PROTO_LITE_TARGETS = [
     "//mediapipe/tasks/cc/core/proto:external_file_java_proto_lite",
 ]
 
+_AUDIO_TASKS_JAVA_PROTO_LITE_TARGETS = [
+    "//mediapipe/tasks/cc/audio/audio_classifier/proto:audio_classifier_graph_options_java_proto_lite",
+]
+
 _VISION_TASKS_JAVA_PROTO_LITE_TARGETS = [
     "//mediapipe/tasks/cc/vision/object_detector/proto:object_detector_options_java_proto_lite",
     "//mediapipe/tasks/cc/vision/image_classifier/proto:image_classifier_graph_options_java_proto_lite",
@@ -62,6 +66,11 @@ def mediapipe_tasks_core_aar(name, srcs, manifest):
             _mediapipe_tasks_java_proto_src_extractor(target = target),
         )
 
+    for target in _AUDIO_TASKS_JAVA_PROTO_LITE_TARGETS:
+        mediapipe_tasks_java_proto_srcs.append(
+            _mediapipe_tasks_java_proto_src_extractor(target = target),
+        )
+
     for target in _VISION_TASKS_JAVA_PROTO_LITE_TARGETS:
         mediapipe_tasks_java_proto_srcs.append(
             _mediapipe_tasks_java_proto_src_extractor(target = target),
@@ -119,9 +128,43 @@ def mediapipe_tasks_core_aar(name, srcs, manifest):
                    "@maven//:com_google_flogger_flogger_system_backend",
                    "@maven//:com_google_code_findbugs_jsr305",
                ] +
+               _AUDIO_TASKS_JAVA_PROTO_LITE_TARGETS +
                _CORE_TASKS_JAVA_PROTO_LITE_TARGETS +
-               _VISION_TASKS_JAVA_PROTO_LITE_TARGETS +
-               _TEXT_TASKS_JAVA_PROTO_LITE_TARGETS,
+               _TEXT_TASKS_JAVA_PROTO_LITE_TARGETS +
+               _VISION_TASKS_JAVA_PROTO_LITE_TARGETS,
+    )
+
+def mediapipe_tasks_audio_aar(name, srcs, native_library):
+    """Builds medaipipe tasks audio AAR.
+
+    Args:
+      name: The bazel target name.
+      srcs: MediaPipe Audio Tasks' source files.
+      native_library: The native library that contains audio tasks' graph and calculators.
+    """
+
+    native.genrule(
+        name = name + "tasks_manifest_generator",
+        outs = ["AndroidManifest.xml"],
+        cmd = """
+cat > $(OUTS) <<EOF
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.tasks.audio">
+    <uses-sdk
+        android:minSdkVersion="24"
+        android:targetSdkVersion="30" />
+</manifest>
+EOF
+""",
+    )
+
+    _mediapipe_tasks_aar(
+        name = name,
+        srcs = srcs,
+        manifest = "AndroidManifest.xml",
+        java_proto_lite_targets = _CORE_TASKS_JAVA_PROTO_LITE_TARGETS + _AUDIO_TASKS_JAVA_PROTO_LITE_TARGETS,
+        native_library = native_library,
     )
 
 def mediapipe_tasks_vision_aar(name, srcs, native_library):
@@ -232,6 +275,7 @@ def _mediapipe_tasks_aar(name, srcs, manifest, java_proto_lite_targets, native_l
             "//mediapipe/framework/formats:landmark_java_proto_lite",
             "//mediapipe/framework/formats:location_data_java_proto_lite",
             "//mediapipe/framework/formats:rect_java_proto_lite",
+            "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:audiodata",
             "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:detection",
             "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category",
             "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:classificationresult",

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD

@@ -111,8 +111,8 @@ android_library(
 android_library(
     name = "gesturerecognizer",
     srcs = [
-        "gesturerecognizer/GestureRecognitionResult.java",
         "gesturerecognizer/GestureRecognizer.java",
+        "gesturerecognizer/GestureRecognizerResult.java",
     ],
     javacopts = [
         "-Xep:AndroidJdkLibsChecker:OFF",

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/core/BaseVisionTaskApi.java

@@ -160,4 +160,18 @@ public class BaseVisionTaskApi implements AutoCloseable {
         .setRotation(-(float) Math.PI * imageProcessingOptions.rotationDegrees() / 180.0f)
         .build();
   }
+
+  /**
+   * Generates the timestamp of a vision task result object from the vision task running mode and
+   * the output packet.
+   *
+   * @param runningMode MediaPipe Vision Tasks {@link RunningMode}.
+   * @param packet the output {@link Packet}.
+   */
+  public static long generateResultTimestampMs(RunningMode runningMode, Packet packet) {
+    if (runningMode == RunningMode.IMAGE) {
+      return -1;
+    }
+    return packet.getTimestamp() / MICROSECONDS_PER_MILLISECOND;
+  }
 }

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java

@@ -64,9 +64,9 @@ import java.util.Optional;
  *       <ul>
  *         <li>The image that gesture recognition runs on.
  *       </ul>
- *   <li>Output GestureRecognitionResult {@link GestureRecognitionResult}
+ *   <li>Output GestureRecognizerResult {@link GestureRecognizerResult}
  *       <ul>
- *         <li>A GestureRecognitionResult containing hand landmarks and recognized hand gestures.
+ *         <li>A GestureRecognizerResult containing hand landmarks and recognized hand gestures.
  *       </ul>
  * </ul>
  */
@@ -152,21 +152,21 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
   public static GestureRecognizer createFromOptions(
       Context context, GestureRecognizerOptions recognizerOptions) {
     // TODO: Consolidate OutputHandler and TaskRunner.
-    OutputHandler<GestureRecognitionResult, MPImage> handler = new OutputHandler<>();
+    OutputHandler<GestureRecognizerResult, MPImage> handler = new OutputHandler<>();
     handler.setOutputPacketConverter(
-        new OutputHandler.OutputPacketConverter<GestureRecognitionResult, MPImage>() {
+        new OutputHandler.OutputPacketConverter<GestureRecognizerResult, MPImage>() {
           @Override
-          public GestureRecognitionResult convertToTaskResult(List<Packet> packets) {
+          public GestureRecognizerResult convertToTaskResult(List<Packet> packets) {
             // If there is no hands detected in the image, just returns empty lists.
             if (packets.get(HAND_GESTURES_OUT_STREAM_INDEX).isEmpty()) {
-              return GestureRecognitionResult.create(
+              return GestureRecognizerResult.create(
                   new ArrayList<>(),
                   new ArrayList<>(),
                   new ArrayList<>(),
                   new ArrayList<>(),
                   packets.get(HAND_GESTURES_OUT_STREAM_INDEX).getTimestamp());
             }
-            return GestureRecognitionResult.create(
+            return GestureRecognizerResult.create(
                 PacketGetter.getProtoVector(
                     packets.get(LANDMARKS_OUT_STREAM_INDEX), NormalizedLandmarkList.parser()),
                 PacketGetter.getProtoVector(
@@ -175,7 +175,8 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
                     packets.get(HANDEDNESS_OUT_STREAM_INDEX), ClassificationList.parser()),
                 PacketGetter.getProtoVector(
                     packets.get(HAND_GESTURES_OUT_STREAM_INDEX), ClassificationList.parser()),
-                packets.get(HAND_GESTURES_OUT_STREAM_INDEX).getTimestamp());
+                BaseVisionTaskApi.generateResultTimestampMs(
+                    recognizerOptions.runningMode(), packets.get(HAND_GESTURES_OUT_STREAM_INDEX)));
           }
 
           @Override
@@ -227,7 +228,7 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
    * @param image a MediaPipe {@link MPImage} object for processing.
    * @throws MediaPipeException if there is an internal error.
    */
-  public GestureRecognitionResult recognize(MPImage image) {
+  public GestureRecognizerResult recognize(MPImage image) {
     return recognize(image, ImageProcessingOptions.builder().build());
   }
 
@@ -251,10 +252,10 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
    *     region-of-interest.
    * @throws MediaPipeException if there is an internal error.
    */
-  public GestureRecognitionResult recognize(
+  public GestureRecognizerResult recognize(
       MPImage image, ImageProcessingOptions imageProcessingOptions) {
     validateImageProcessingOptions(imageProcessingOptions);
-    return (GestureRecognitionResult) processImageData(image, imageProcessingOptions);
+    return (GestureRecognizerResult) processImageData(image, imageProcessingOptions);
   }
 
   /**
@@ -275,7 +276,7 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
    * @param timestampMs the input timestamp (in milliseconds).
    * @throws MediaPipeException if there is an internal error.
    */
-  public GestureRecognitionResult recognizeForVideo(MPImage image, long timestampMs) {
+  public GestureRecognizerResult recognizeForVideo(MPImage image, long timestampMs) {
     return recognizeForVideo(image, ImageProcessingOptions.builder().build(), timestampMs);
   }
 
@@ -302,10 +303,10 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
    *     region-of-interest.
    * @throws MediaPipeException if there is an internal error.
    */
-  public GestureRecognitionResult recognizeForVideo(
+  public GestureRecognizerResult recognizeForVideo(
       MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
     validateImageProcessingOptions(imageProcessingOptions);
-    return (GestureRecognitionResult) processVideoData(image, imageProcessingOptions, timestampMs);
+    return (GestureRecognizerResult) processVideoData(image, imageProcessingOptions, timestampMs);
   }
 
   /**
@@ -424,7 +425,7 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
        * recognizer is in the live stream mode.
        */
       public abstract Builder setResultListener(
-          ResultListener<GestureRecognitionResult, MPImage> value);
+          ResultListener<GestureRecognizerResult, MPImage> value);
 
       /** Sets an optional error listener. */
       public abstract Builder setErrorListener(ErrorListener value);
@@ -471,7 +472,7 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
 
     abstract Optional<ClassifierOptions> customGesturesClassifierOptions();
 
-    abstract Optional<ResultListener<GestureRecognitionResult, MPImage>> resultListener();
+    abstract Optional<ResultListener<GestureRecognizerResult, MPImage>> resultListener();
 
     abstract Optional<ErrorListener> errorListener();
 

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerResult.java

@@ -29,20 +29,20 @@ import java.util.List;
 
 /** Represents the gesture recognition results generated by {@link GestureRecognizer}. */
 @AutoValue
-public abstract class GestureRecognitionResult implements TaskResult {
+public abstract class GestureRecognizerResult implements TaskResult {
 
   private static final int kGestureDefaultIndex = -1;
 
   /**
-   * Creates a {@link GestureRecognitionResult} instance from the lists of landmarks, handedness,
-   * and gestures protobuf messages.
+   * Creates a {@link GestureRecognizerResult} instance from the lists of landmarks, handedness, and
+   * gestures protobuf messages.
    *
    * @param landmarksProto a List of {@link NormalizedLandmarkList}
    * @param worldLandmarksProto a List of {@link LandmarkList}
    * @param handednessesProto a List of {@link ClassificationList}
    * @param gesturesProto a List of {@link ClassificationList}
    */
-  static GestureRecognitionResult create(
+  static GestureRecognizerResult create(
       List<NormalizedLandmarkList> landmarksProto,
       List<LandmarkList> worldLandmarksProto,
       List<ClassificationList> handednessesProto,
@@ -106,7 +106,7 @@ public abstract class GestureRecognitionResult implements TaskResult {
                 classification.getDisplayName()));
       }
     }
-    return new AutoValue_GestureRecognitionResult(
+    return new AutoValue_GestureRecognizerResult(
         timestampMs,
         Collections.unmodifiableList(multiHandLandmarks),
         Collections.unmodifiableList(multiHandWorldLandmarks),

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarker.java

@@ -165,7 +165,8 @@ public final class HandLandmarker extends BaseVisionTaskApi {
                     packets.get(WORLD_LANDMARKS_OUT_STREAM_INDEX), LandmarkList.parser()),
                 PacketGetter.getProtoVector(
                     packets.get(HANDEDNESS_OUT_STREAM_INDEX), ClassificationList.parser()),
-                packets.get(LANDMARKS_OUT_STREAM_INDEX).getTimestamp());
+                BaseVisionTaskApi.generateResultTimestampMs(
+                    landmarkerOptions.runningMode(), packets.get(LANDMARKS_OUT_STREAM_INDEX)));
           }
 
           @Override
@@ -241,8 +242,7 @@ public final class HandLandmarker extends BaseVisionTaskApi {
    *     region-of-interest.
    * @throws MediaPipeException if there is an internal error.
    */
-  public HandLandmarkerResult detect(
-      MPImage image, ImageProcessingOptions imageProcessingOptions) {
+  public HandLandmarkerResult detect(MPImage image, ImageProcessingOptions imageProcessingOptions) {
     validateImageProcessingOptions(imageProcessingOptions);
     return (HandLandmarkerResult) processImageData(image, imageProcessingOptions);
   }
@@ -295,8 +295,7 @@ public final class HandLandmarker extends BaseVisionTaskApi {
   public HandLandmarkerResult detectForVideo(
       MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
     validateImageProcessingOptions(imageProcessingOptions);
-    return (HandLandmarkerResult)
-        processVideoData(image, imageProcessingOptions, timestampMs);
+    return (HandLandmarkerResult) processVideoData(image, imageProcessingOptions, timestampMs);
   }
 
   /**

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageclassifier/ImageClassifier.java

@@ -176,7 +176,8 @@ public final class ImageClassifier extends BaseVisionTaskApi {
                       PacketGetter.getProto(
                           packets.get(CLASSIFICATIONS_OUT_STREAM_INDEX),
                           ClassificationsProto.ClassificationResult.getDefaultInstance())),
-                  packets.get(CLASSIFICATIONS_OUT_STREAM_INDEX).getTimestamp());
+                  BaseVisionTaskApi.generateResultTimestampMs(
+                      options.runningMode(), packets.get(CLASSIFICATIONS_OUT_STREAM_INDEX)));
             } catch (IOException e) {
               throw new MediaPipeException(
                   MediaPipeException.StatusCode.INTERNAL.ordinal(), e.getMessage());

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java

@@ -173,7 +173,8 @@ public final class ObjectDetector extends BaseVisionTaskApi {
             return ObjectDetectionResult.create(
                 PacketGetter.getProtoVector(
                     packets.get(DETECTIONS_OUT_STREAM_INDEX), Detection.parser()),
-                packets.get(DETECTIONS_OUT_STREAM_INDEX).getTimestamp());
+                BaseVisionTaskApi.generateResultTimestampMs(
+                    detectorOptions.runningMode(), packets.get(DETECTIONS_OUT_STREAM_INDEX)));
           }
 
           @Override

mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/utils/AndroidManifest.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.tasks.components.utilstest"
+    android:versionCode="1"
+    android:versionName="1.0" >
+
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
+
+    <uses-sdk android:minSdkVersion="24"
+        android:targetSdkVersion="30" />
+
+    <application
+        android:label="utilstest"
+        android:name="android.support.multidex.MultiDexApplication"
+        android:taskAffinity="">
+        <uses-library android:name="android.test.runner" />
+    </application>
+
+    <instrumentation
+        android:name="com.google.android.apps.common.testing.testrunner.GoogleInstrumentationTestRunner"
+        android:targetPackage="com.google.mediapipe.tasks.components.utilstest" />
+
+</manifest>

mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/utils/BUILD

@@ -0,0 +1,19 @@
+# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+# TODO: Enable this in OSS

mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/utils/CosineSimilarityTest.java

@@ -0,0 +1,116 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.components.utils;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.mediapipe.tasks.components.containers.Embedding;
+import java.util.Optional;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Tests for {@link CosineSimilarity}. */
+@RunWith(AndroidJUnit4.class)
+public final class CosineSimilarityTest {
+
+  @Test
+  public void failsWithQuantizedAndFloatEmbeddings() {
+    Embedding u =
+        Embedding.create(
+            new float[] {1.0f}, new byte[0], /*headIndex=*/ 0, /*headName=*/ Optional.empty());
+    Embedding v =
+        Embedding.create(
+            new float[0], new byte[] {1}, /*headIndex=*/ 0, /*headName=*/ Optional.empty());
+
+    IllegalArgumentException exception =
+        assertThrows(IllegalArgumentException.class, () -> CosineSimilarity.compute(u, v));
+    assertThat(exception)
+        .hasMessageThat()
+        .contains("Cannot compute cosine similarity between quantized and float embeddings");
+  }
+
+  @Test
+  public void failsWithZeroNorm() {
+    Embedding u =
+        Embedding.create(
+            new float[] {0.0f}, new byte[0], /*headIndex=*/ 0, /*headName=*/ Optional.empty());
+
+    IllegalArgumentException exception =
+        assertThrows(IllegalArgumentException.class, () -> CosineSimilarity.compute(u, u));
+    assertThat(exception)
+        .hasMessageThat()
+        .contains("Cannot compute cosine similarity on embedding with 0 norm");
+  }
+
+  @Test
+  public void failsWithDifferentSizes() {
+    Embedding u =
+        Embedding.create(
+            new float[] {1.0f, 2.0f},
+            new byte[0],
+            /*headIndex=*/ 0,
+            /*headName=*/ Optional.empty());
+    Embedding v =
+        Embedding.create(
+            new float[] {1.0f, 2.0f, 3.0f},
+            new byte[0],
+            /*headIndex=*/ 0,
+            /*headName=*/ Optional.empty());
+
+    IllegalArgumentException exception =
+        assertThrows(IllegalArgumentException.class, () -> CosineSimilarity.compute(u, v));
+    assertThat(exception)
+        .hasMessageThat()
+        .contains("Cannot compute cosine similarity between embeddings of different sizes");
+  }
+
+  @Test
+  public void succeedsWithFloatEmbeddings() {
+    Embedding u =
+        Embedding.create(
+            new float[] {1.0f, 0.0f, 0.0f, 0.0f},
+            new byte[0],
+            /*headIndex=*/ 0,
+            /*headName=*/ Optional.empty());
+    Embedding v =
+        Embedding.create(
+            new float[] {0.5f, 0.5f, 0.5f, 0.5f},
+            new byte[0],
+            /*headIndex=*/ 0,
+            /*headName=*/ Optional.empty());
+
+    assertThat(CosineSimilarity.compute(u, v)).isEqualTo(0.5);
+  }
+
+  @Test
+  public void succeedsWithQuantizedEmbeddings() {
+    Embedding u =
+        Embedding.create(
+            new float[0],
+            new byte[] {127, 0, 0, 0},
+            /*headIndex=*/ 0,
+            /*headName=*/ Optional.empty());
+    Embedding v =
+        Embedding.create(
+            new float[0],
+            new byte[] {-128, 0, 0, 0},
+            /*headIndex=*/ 0,
+            /*headName=*/ Optional.empty());
+
+    assertThat(CosineSimilarity.compute(u, v)).isEqualTo(-1.0);
+  }
+}

mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java

@@ -80,10 +80,10 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(THUMB_UP_IMAGE));
-      GestureRecognitionResult expectedResult =
-          getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
+      GestureRecognizerResult expectedResult =
+          getExpectedGestureRecognizerResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
       assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
     }
 
@@ -98,7 +98,7 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(NO_HANDS_IMAGE));
       assertThat(actualResult.landmarks()).isEmpty();
       assertThat(actualResult.worldLandmarks()).isEmpty();
@@ -119,10 +119,10 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(THUMB_UP_IMAGE));
-      GestureRecognitionResult expectedResult =
-          getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
+      GestureRecognizerResult expectedResult =
+          getExpectedGestureRecognizerResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
       // Only contains one top scoring gesture.
       assertThat(actualResult.gestures().get(0)).hasSize(1);
       assertActualGestureEqualExpectedGesture(
@@ -141,7 +141,7 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(TWO_HANDS_IMAGE));
       assertThat(actualResult.handednesses()).hasSize(2);
     }
@@ -160,7 +160,7 @@ public class GestureRecognizerTest {
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
       ImageProcessingOptions imageProcessingOptions =
           ImageProcessingOptions.builder().setRotationDegrees(-90).build();
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(
               getImageFromAsset(POINTING_UP_ROTATED_IMAGE), imageProcessingOptions);
       assertThat(actualResult.gestures()).hasSize(1);
@@ -179,10 +179,10 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
-      GestureRecognitionResult expectedResult =
-          getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
+      GestureRecognizerResult expectedResult =
+          getExpectedGestureRecognizerResult(FIST_LANDMARKS, FIST_LABEL);
       assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
     }
 
@@ -199,10 +199,10 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
-      GestureRecognitionResult expectedResult =
-          getExpectedGestureRecognitionResult(FIST_LANDMARKS, ROCK_LABEL);
+      GestureRecognizerResult expectedResult =
+          getExpectedGestureRecognizerResult(FIST_LANDMARKS, ROCK_LABEL);
       assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
     }
 
@@ -223,10 +223,10 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
-      GestureRecognitionResult expectedResult =
-          getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
+      GestureRecognizerResult expectedResult =
+          getExpectedGestureRecognizerResult(FIST_LANDMARKS, FIST_LABEL);
       assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
     }
 
@@ -247,7 +247,7 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
       assertThat(actualResult.landmarks()).isEmpty();
       assertThat(actualResult.worldLandmarks()).isEmpty();
@@ -280,7 +280,7 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
       assertThat(actualResult.landmarks()).isEmpty();
       assertThat(actualResult.worldLandmarks()).isEmpty();
@@ -306,10 +306,10 @@ public class GestureRecognizerTest {
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
-      GestureRecognitionResult expectedResult =
-          getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
+      GestureRecognizerResult expectedResult =
+          getExpectedGestureRecognizerResult(FIST_LANDMARKS, FIST_LABEL);
       assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
     }
 
@@ -478,10 +478,10 @@ public class GestureRecognizerTest {
 
     GestureRecognizer gestureRecognizer =
         GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-    GestureRecognitionResult actualResult =
+    GestureRecognizerResult actualResult =
         gestureRecognizer.recognize(getImageFromAsset(THUMB_UP_IMAGE));
-    GestureRecognitionResult expectedResult =
-        getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
+    GestureRecognizerResult expectedResult =
+        getExpectedGestureRecognizerResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
     assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
   }
 
@@ -497,10 +497,10 @@ public class GestureRecognizerTest {
             .build();
     GestureRecognizer gestureRecognizer =
         GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
-    GestureRecognitionResult expectedResult =
-        getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
+    GestureRecognizerResult expectedResult =
+        getExpectedGestureRecognizerResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
     for (int i = 0; i < 3; i++) {
-      GestureRecognitionResult actualResult =
+      GestureRecognizerResult actualResult =
           gestureRecognizer.recognizeForVideo(
               getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ i);
       assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
@@ -510,8 +510,8 @@ public class GestureRecognizerTest {
   @Test
   public void recognize_failsWithOutOfOrderInputTimestamps() throws Exception {
     MPImage image = getImageFromAsset(THUMB_UP_IMAGE);
-    GestureRecognitionResult expectedResult =
-        getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
+    GestureRecognizerResult expectedResult =
+        getExpectedGestureRecognizerResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
     GestureRecognizerOptions options =
         GestureRecognizerOptions.builder()
             .setBaseOptions(
@@ -542,8 +542,8 @@ public class GestureRecognizerTest {
   @Test
   public void recognize_successWithLiveSteamMode() throws Exception {
     MPImage image = getImageFromAsset(THUMB_UP_IMAGE);
-    GestureRecognitionResult expectedResult =
-        getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
+    GestureRecognizerResult expectedResult =
+        getExpectedGestureRecognizerResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL);
     GestureRecognizerOptions options =
         GestureRecognizerOptions.builder()
             .setBaseOptions(
@@ -572,7 +572,7 @@ public class GestureRecognizerTest {
     return new BitmapImageBuilder(BitmapFactory.decodeStream(istr)).build();
   }
 
-  private static GestureRecognitionResult getExpectedGestureRecognitionResult(
+  private static GestureRecognizerResult getExpectedGestureRecognizerResult(
       String filePath, String gestureLabel) throws Exception {
     AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets();
     InputStream istr = assetManager.open(filePath);
@@ -583,7 +583,7 @@ public class GestureRecognizerTest {
             .addClassification(
                 ClassificationProto.Classification.newBuilder().setLabel(gestureLabel))
             .build();
-    return GestureRecognitionResult.create(
+    return GestureRecognizerResult.create(
         Arrays.asList(landmarksDetectionResultProto.getLandmarks()),
         Arrays.asList(landmarksDetectionResultProto.getWorldLandmarks()),
         Arrays.asList(landmarksDetectionResultProto.getClassifications()),
@@ -592,7 +592,7 @@ public class GestureRecognizerTest {
   }
 
   private static void assertActualResultApproximatelyEqualsToExpectedResult(
-      GestureRecognitionResult actualResult, GestureRecognitionResult expectedResult) {
+      GestureRecognizerResult actualResult, GestureRecognizerResult expectedResult) {
     // Expects to have the same number of hands detected.
     assertThat(actualResult.landmarks()).hasSize(expectedResult.landmarks().size());
     assertThat(actualResult.worldLandmarks()).hasSize(expectedResult.worldLandmarks().size());

mediapipe/tasks/python/components/containers/BUILD

@@ -85,21 +85,12 @@ py_library(
     ],
 )
 
-py_library(
-    name = "classifications",
-    srcs = ["classifications.py"],
-    deps = [
-        ":category",
-        "//mediapipe/tasks/cc/components/containers/proto:classifications_py_pb2",
-        "//mediapipe/tasks/python/core:optional_dependencies",
-    ],
-)
-
 py_library(
     name = "classification_result",
     srcs = ["classification_result.py"],
     deps = [
         ":category",
+        "//mediapipe/framework/formats:classification_py_pb2",
         "//mediapipe/tasks/cc/components/containers/proto:classifications_py_pb2",
         "//mediapipe/tasks/python/core:optional_dependencies",
     ],

mediapipe/tasks/python/components/containers/classification_result.py

@@ -16,10 +16,13 @@
 import dataclasses
 from typing import List, Optional
 
+from mediapipe.framework.formats import classification_pb2
 from mediapipe.tasks.cc.components.containers.proto import classifications_pb2
 from mediapipe.tasks.python.components.containers import category as category_module
 from mediapipe.tasks.python.core.optional_dependencies import doc_controls
 
+_ClassificationProto = classification_pb2.Classification
+_ClassificationListProto = classification_pb2.ClassificationList
 _ClassificationsProto = classifications_pb2.Classifications
 _ClassificationResultProto = classifications_pb2.ClassificationResult
 
@@ -41,6 +44,22 @@ class Classifications:
   head_index: int
   head_name: Optional[str] = None
 
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _ClassificationsProto:
+    """Generates a Classifications protobuf object."""
+    classification_list_proto = _ClassificationListProto()
+    for category in self.categories:
+      classification_proto = _ClassificationProto(
+          index=category.index,
+          score=category.score,
+          label=category.category_name,
+          display_name=category.display_name)
+      classification_list_proto.classification.append(classification_proto)
+    return _ClassificationsProto(
+        classification_list=classification_list_proto,
+        head_index=self.head_index,
+        head_name=self.head_name)
+
   @classmethod
   @doc_controls.do_not_generate_docs
   def create_from_pb2(cls, pb2_obj: _ClassificationsProto) -> 'Classifications':
@@ -78,6 +97,15 @@ class ClassificationResult:
   classifications: List[Classifications]
   timestamp_ms: Optional[int] = None
 
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _ClassificationResultProto:
+    """Generates a ClassificationResult protobuf object."""
+    return _ClassificationResultProto(
+        classifications=[
+            classification.to_pb2() for classification in self.classifications
+        ],
+        timestamp_ms=self.timestamp_ms)
+
   @classmethod
   @doc_controls.do_not_generate_docs
   def create_from_pb2(

mediapipe/tasks/python/components/containers/classifications.py

@@ -1,168 +0,0 @@
-# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Classifications data class."""
-
-import dataclasses
-from typing import Any, List, Optional
-
-from mediapipe.tasks.cc.components.containers.proto import classifications_pb2
-from mediapipe.tasks.python.components.containers import category as category_module
-from mediapipe.tasks.python.core.optional_dependencies import doc_controls
-
-_ClassificationEntryProto = classifications_pb2.ClassificationEntry
-_ClassificationsProto = classifications_pb2.Classifications
-_ClassificationResultProto = classifications_pb2.ClassificationResult
-
-
-@dataclasses.dataclass
-class ClassificationEntry:
-  """List of predicted classes (aka labels) for a given classifier head.
-
-  Attributes:
-    categories: The array of predicted categories, usually sorted by descending
-      scores (e.g. from high to low probability).
-    timestamp_ms: The optional timestamp (in milliseconds) associated to the
-      classification entry. This is useful for time series use cases, e.g.,
-      audio classification.
-  """
-
-  categories: List[category_module.Category]
-  timestamp_ms: Optional[int] = None
-
-  @doc_controls.do_not_generate_docs
-  def to_pb2(self) -> _ClassificationEntryProto:
-    """Generates a ClassificationEntry protobuf object."""
-    return _ClassificationEntryProto(
-        categories=[category.to_pb2() for category in self.categories],
-        timestamp_ms=self.timestamp_ms)
-
-  @classmethod
-  @doc_controls.do_not_generate_docs
-  def create_from_pb2(
-      cls, pb2_obj: _ClassificationEntryProto) -> 'ClassificationEntry':
-    """Creates a `ClassificationEntry` object from the given protobuf object."""
-    return ClassificationEntry(
-        categories=[
-            category_module.Category.create_from_pb2(category)
-            for category in pb2_obj.categories
-        ],
-        timestamp_ms=pb2_obj.timestamp_ms)
-
-  def __eq__(self, other: Any) -> bool:
-    """Checks if this object is equal to the given object.
-
-    Args:
-      other: The object to be compared with.
-
-    Returns:
-      True if the objects are equal.
-    """
-    if not isinstance(other, ClassificationEntry):
-      return False
-
-    return self.to_pb2().__eq__(other.to_pb2())
-
-
-@dataclasses.dataclass
-class Classifications:
-  """Represents the classifications for a given classifier head.
-
-  Attributes:
-    entries: A list of `ClassificationEntry` objects.
-    head_index: The index of the classifier head these categories refer to. This
-      is useful for multi-head models.
-    head_name: The name of the classifier head, which is the corresponding
-      tensor metadata name.
-  """
-
-  entries: List[ClassificationEntry]
-  head_index: int
-  head_name: str
-
-  @doc_controls.do_not_generate_docs
-  def to_pb2(self) -> _ClassificationsProto:
-    """Generates a Classifications protobuf object."""
-    return _ClassificationsProto(
-        entries=[entry.to_pb2() for entry in self.entries],
-        head_index=self.head_index,
-        head_name=self.head_name)
-
-  @classmethod
-  @doc_controls.do_not_generate_docs
-  def create_from_pb2(cls, pb2_obj: _ClassificationsProto) -> 'Classifications':
-    """Creates a `Classifications` object from the given protobuf object."""
-    return Classifications(
-        entries=[
-            ClassificationEntry.create_from_pb2(entry)
-            for entry in pb2_obj.entries
-        ],
-        head_index=pb2_obj.head_index,
-        head_name=pb2_obj.head_name)
-
-  def __eq__(self, other: Any) -> bool:
-    """Checks if this object is equal to the given object.
-
-    Args:
-      other: The object to be compared with.
-
-    Returns:
-      True if the objects are equal.
-    """
-    if not isinstance(other, Classifications):
-      return False
-
-    return self.to_pb2().__eq__(other.to_pb2())
-
-
-@dataclasses.dataclass
-class ClassificationResult:
-  """Contains one set of results per classifier head.
-
-  Attributes:
-    classifications: A list of `Classifications` objects.
-  """
-
-  classifications: List[Classifications]
-
-  @doc_controls.do_not_generate_docs
-  def to_pb2(self) -> _ClassificationResultProto:
-    """Generates a ClassificationResult protobuf object."""
-    return _ClassificationResultProto(classifications=[
-        classification.to_pb2() for classification in self.classifications
-    ])
-
-  @classmethod
-  @doc_controls.do_not_generate_docs
-  def create_from_pb2(
-      cls, pb2_obj: _ClassificationResultProto) -> 'ClassificationResult':
-    """Creates a `ClassificationResult` object from the given protobuf object.
-    """
-    return ClassificationResult(classifications=[
-        Classifications.create_from_pb2(classification)
-        for classification in pb2_obj.classifications
-    ])
-
-  def __eq__(self, other: Any) -> bool:
-    """Checks if this object is equal to the given object.
-
-    Args:
-      other: The object to be compared with.
-
-    Returns:
-      True if the objects are equal.
-    """
-    if not isinstance(other, ClassificationResult):
-      return False
-
-    return self.to_pb2().__eq__(other.to_pb2())

mediapipe/tasks/python/test/text/BUILD

@@ -27,7 +27,7 @@ py_test(
     ],
     deps = [
         "//mediapipe/tasks/python/components/containers:category",
-        "//mediapipe/tasks/python/components/containers:classifications",
+        "//mediapipe/tasks/python/components/containers:classification_result",
         "//mediapipe/tasks/python/components/processors:classifier_options",
         "//mediapipe/tasks/python/core:base_options",
         "//mediapipe/tasks/python/test:test_utils",

mediapipe/tasks/python/test/text/text_classifier_test.py

@@ -20,18 +20,17 @@ from absl.testing import absltest
 from absl.testing import parameterized
 
 from mediapipe.tasks.python.components.containers import category
-from mediapipe.tasks.python.components.containers import classifications as classifications_module
+from mediapipe.tasks.python.components.containers import classification_result as classification_result_module
 from mediapipe.tasks.python.components.processors import classifier_options
 from mediapipe.tasks.python.core import base_options as base_options_module
 from mediapipe.tasks.python.test import test_utils
 from mediapipe.tasks.python.text import text_classifier
 
+TextClassifierResult = classification_result_module.ClassificationResult
 _BaseOptions = base_options_module.BaseOptions
 _ClassifierOptions = classifier_options.ClassifierOptions
 _Category = category.Category
-_ClassificationEntry = classifications_module.ClassificationEntry
-_Classifications = classifications_module.Classifications
-_TextClassifierResult = classifications_module.ClassificationResult
+_Classifications = classification_result_module.Classifications
 _TextClassifier = text_classifier.TextClassifier
 _TextClassifierOptions = text_classifier.TextClassifierOptions
 
@@ -43,90 +42,82 @@ _NEGATIVE_TEXT = 'What a waste of my time.'
 _POSITIVE_TEXT = ('This is the best movie I’ve seen in recent years.'
                   'Strongly recommend it!')
 
-_BERT_NEGATIVE_RESULTS = _TextClassifierResult(classifications=[
-    _Classifications(
-        entries=[
-            _ClassificationEntry(
-                categories=[
-                    _Category(
-                        index=0,
-                        score=0.999479,
-                        display_name='',
-                        category_name='negative'),
-                    _Category(
-                        index=1,
-                        score=0.00052154,
-                        display_name='',
-                        category_name='positive')
-                ],
-                timestamp_ms=0)
-        ],
-        head_index=0,
-        head_name='probability')
-])
-_BERT_POSITIVE_RESULTS = _TextClassifierResult(classifications=[
-    _Classifications(
-        entries=[
-            _ClassificationEntry(
-                categories=[
-                    _Category(
-                        index=1,
-                        score=0.999466,
-                        display_name='',
-                        category_name='positive'),
-                    _Category(
-                        index=0,
-                        score=0.000533596,
-                        display_name='',
-                        category_name='negative')
-                ],
-                timestamp_ms=0)
-        ],
-        head_index=0,
-        head_name='probability')
-])
-_REGEX_NEGATIVE_RESULTS = _TextClassifierResult(classifications=[
-    _Classifications(
-        entries=[
-            _ClassificationEntry(
-                categories=[
-                    _Category(
-                        index=0,
-                        score=0.81313,
-                        display_name='',
-                        category_name='Negative'),
-                    _Category(
-                        index=1,
-                        score=0.1868704,
-                        display_name='',
-                        category_name='Positive')
-                ],
-                timestamp_ms=0)
-        ],
-        head_index=0,
-        head_name='probability')
-])
-_REGEX_POSITIVE_RESULTS = _TextClassifierResult(classifications=[
-    _Classifications(
-        entries=[
-            _ClassificationEntry(
-                categories=[
-                    _Category(
-                        index=1,
-                        score=0.5134273,
-                        display_name='',
-                        category_name='Positive'),
-                    _Category(
-                        index=0,
-                        score=0.486573,
-                        display_name='',
-                        category_name='Negative')
-                ],
-                timestamp_ms=0)
-        ],
-        head_index=0,
-        head_name='probability')
-])
+_BERT_NEGATIVE_RESULTS = TextClassifierResult(
+    classifications=[
+        _Classifications(
+            categories=[
+                _Category(
+                    index=0,
+                    score=0.999479,
+                    display_name='',
+                    category_name='negative'),
+                _Category(
+                    index=1,
+                    score=0.00052154,
+                    display_name='',
+                    category_name='positive')
+            ],
+            head_index=0,
+            head_name='probability')
+    ],
+    timestamp_ms=0)
+_BERT_POSITIVE_RESULTS = TextClassifierResult(
+    classifications=[
+        _Classifications(
+            categories=[
+                _Category(
+                    index=1,
+                    score=0.999466,
+                    display_name='',
+                    category_name='positive'),
+                _Category(
+                    index=0,
+                    score=0.000533596,
+                    display_name='',
+                    category_name='negative')
+            ],
+            head_index=0,
+            head_name='probability')
+    ],
+    timestamp_ms=0)
+_REGEX_NEGATIVE_RESULTS = TextClassifierResult(
+    classifications=[
+        _Classifications(
+            categories=[
+                _Category(
+                    index=0,
+                    score=0.81313,
+                    display_name='',
+                    category_name='Negative'),
+                _Category(
+                    index=1,
+                    score=0.1868704,
+                    display_name='',
+                    category_name='Positive')
+            ],
+            head_index=0,
+            head_name='probability')
+    ],
+    timestamp_ms=0)
+_REGEX_POSITIVE_RESULTS = TextClassifierResult(
+    classifications=[
+        _Classifications(
+            categories=[
+                _Category(
+                    index=1,
+                    score=0.5134273,
+                    display_name='',
+                    category_name='Positive'),
+                _Category(
+                    index=0,
+                    score=0.486573,
+                    display_name='',
+                    category_name='Negative')
+            ],
+            head_index=0,
+            head_name='probability')
+    ],
+    timestamp_ms=0)
 
 
 class ModelFileType(enum.Enum):

mediapipe/tasks/python/test/vision/BUILD

@@ -47,7 +47,7 @@ py_test(
     deps = [
         "//mediapipe/python:_framework_bindings",
         "//mediapipe/tasks/python/components/containers:category",
-        "//mediapipe/tasks/python/components/containers:classifications",
+        "//mediapipe/tasks/python/components/containers:classification_result",
         "//mediapipe/tasks/python/components/containers:rect",
         "//mediapipe/tasks/python/components/processors:classifier_options",
         "//mediapipe/tasks/python/core:base_options",

mediapipe/tasks/python/test/vision/image_classifier_test.py

@@ -23,8 +23,8 @@ from absl.testing import parameterized
 import numpy as np
 
 from mediapipe.python._framework_bindings import image
-from mediapipe.tasks.python.components.containers import category
-from mediapipe.tasks.python.components.containers import classifications as classifications_module
+from mediapipe.tasks.python.components.containers import category as category_module
+from mediapipe.tasks.python.components.containers import classification_result as classification_result_module
 from mediapipe.tasks.python.components.containers import rect
 from mediapipe.tasks.python.components.processors import classifier_options
 from mediapipe.tasks.python.core import base_options as base_options_module
@@ -33,13 +33,12 @@ from mediapipe.tasks.python.vision import image_classifier
 from mediapipe.tasks.python.vision.core import image_processing_options as image_processing_options_module
 from mediapipe.tasks.python.vision.core import vision_task_running_mode
 
+ImageClassifierResult = classification_result_module.ClassificationResult
 _Rect = rect.Rect
 _BaseOptions = base_options_module.BaseOptions
 _ClassifierOptions = classifier_options.ClassifierOptions
-_Category = category.Category
-_ClassificationEntry = classifications_module.ClassificationEntry
-_Classifications = classifications_module.Classifications
-_ClassificationResult = classifications_module.ClassificationResult
+_Category = category_module.Category
+_Classifications = classification_result_module.Classifications
 _Image = image.Image
 _ImageClassifier = image_classifier.ImageClassifier
 _ImageClassifierOptions = image_classifier.ImageClassifierOptions
@@ -55,68 +54,62 @@ _MAX_RESULTS = 3
 _TEST_DATA_DIR = 'mediapipe/tasks/testdata/vision'
 
 
-def _generate_empty_results(timestamp_ms: int) -> _ClassificationResult:
-  return _ClassificationResult(classifications=[
-      _Classifications(
-          entries=[
-              _ClassificationEntry(categories=[], timestamp_ms=timestamp_ms)
-          ],
-          head_index=0,
-          head_name='probability')
-  ])
+def _generate_empty_results() -> ImageClassifierResult:
+  return ImageClassifierResult(
+      classifications=[
+          _Classifications(
+              categories=[], head_index=0, head_name='probability')
+      ],
+      timestamp_ms=0)
 
 
-def _generate_burger_results(timestamp_ms: int) -> _ClassificationResult:
-  return _ClassificationResult(classifications=[
-      _Classifications(
-          entries=[
-              _ClassificationEntry(
-                  categories=[
-                      _Category(
-                          index=934,
-                          score=0.793959,
-                          display_name='',
-                          category_name='cheeseburger'),
-                      _Category(
-                          index=932,
-                          score=0.0273929,
-                          display_name='',
-                          category_name='bagel'),
-                      _Category(
-                          index=925,
-                          score=0.0193408,
-                          display_name='',
-                          category_name='guacamole'),
-                      _Category(
-                          index=963,
-                          score=0.00632786,
-                          display_name='',
-                          category_name='meat loaf')
-                  ],
-                  timestamp_ms=timestamp_ms)
-          ],
-          head_index=0,
-          head_name='probability')
-  ])
+def _generate_burger_results() -> ImageClassifierResult:
+  return ImageClassifierResult(
+      classifications=[
+          _Classifications(
+              categories=[
+                  _Category(
+                      index=934,
+                      score=0.793959,
+                      display_name='',
+                      category_name='cheeseburger'),
+                  _Category(
+                      index=932,
+                      score=0.0273929,
+                      display_name='',
+                      category_name='bagel'),
+                  _Category(
+                      index=925,
+                      score=0.0193408,
+                      display_name='',
+                      category_name='guacamole'),
+                  _Category(
+                      index=963,
+                      score=0.00632786,
+                      display_name='',
+                      category_name='meat loaf')
+              ],
+              head_index=0,
+              head_name='probability')
+      ],
+      timestamp_ms=0)
 
 
-def _generate_soccer_ball_results(timestamp_ms: int) -> _ClassificationResult:
-  return _ClassificationResult(classifications=[
-      _Classifications(
-          entries=[
-              _ClassificationEntry(
-                  categories=[
-                      _Category(
-                          index=806,
-                          score=0.996527,
-                          display_name='',
-                          category_name='soccer ball')
-                  ],
-                  timestamp_ms=timestamp_ms)
-          ],
-          head_index=0,
-          head_name='probability')
-  ])
+def _generate_soccer_ball_results() -> ImageClassifierResult:
+  return ImageClassifierResult(
+      classifications=[
+          _Classifications(
+              categories=[
+                  _Category(
+                      index=806,
+                      score=0.996527,
+                      display_name='',
+                      category_name='soccer ball')
+              ],
+              head_index=0,
+              head_name='probability')
+      ],
+      timestamp_ms=0)
 
 
 class ModelFileType(enum.Enum):
@@ -163,8 +156,8 @@ class ImageClassifierTest(parameterized.TestCase):
       self.assertIsInstance(classifier, _ImageClassifier)
 
   @parameterized.parameters(
-      (ModelFileType.FILE_NAME, 4, _generate_burger_results(0)),
-      (ModelFileType.FILE_CONTENT, 4, _generate_burger_results(0)))
+      (ModelFileType.FILE_NAME, 4, _generate_burger_results()),
+      (ModelFileType.FILE_CONTENT, 4, _generate_burger_results()))
   def test_classify(self, model_file_type, max_results,
                     expected_classification_result):
     # Creates classifier.
@@ -193,8 +186,8 @@ class ImageClassifierTest(parameterized.TestCase):
     classifier.close()
 
   @parameterized.parameters(
-      (ModelFileType.FILE_NAME, 4, _generate_burger_results(0)),
-      (ModelFileType.FILE_CONTENT, 4, _generate_burger_results(0)))
+      (ModelFileType.FILE_NAME, 4, _generate_burger_results()),
+      (ModelFileType.FILE_CONTENT, 4, _generate_burger_results()))
   def test_classify_in_context(self, model_file_type, max_results,
                                expected_classification_result):
     if model_file_type is ModelFileType.FILE_NAME:
@@ -234,7 +227,7 @@ class ImageClassifierTest(parameterized.TestCase):
       image_result = classifier.classify(test_image, image_processing_options)
       # Comparing results.
       test_utils.assert_proto_equals(self, image_result.to_pb2(),
-                                     _generate_soccer_ball_results(0).to_pb2())
+                                     _generate_soccer_ball_results().to_pb2())
 
   def test_score_threshold_option(self):
     custom_classifier_options = _ClassifierOptions(
@@ -248,8 +241,8 @@ class ImageClassifierTest(parameterized.TestCase):
       classifications = image_result.classifications
 
       for classification in classifications:
-        for entry in classification.entries:
-          score = entry.categories[0].score
+        for category in classification.categories:
+          score = category.score
           self.assertGreaterEqual(
               score, _SCORE_THRESHOLD,
               f'Classification with score lower than threshold found. '
@@ -264,7 +257,7 @@ class ImageClassifierTest(parameterized.TestCase):
     with _ImageClassifier.create_from_options(options) as classifier:
       # Performs image classification on the input.
       image_result = classifier.classify(self.test_image)
-      categories = image_result.classifications[0].entries[0].categories
+      categories = image_result.classifications[0].categories
 
       self.assertLessEqual(
           len(categories), _MAX_RESULTS, 'Too many results returned.')
@@ -281,8 +274,8 @@ class ImageClassifierTest(parameterized.TestCase):
       classifications = image_result.classifications
 
       for classification in classifications:
-        for entry in classification.entries:
-          label = entry.categories[0].category_name
+        for category in classification.categories:
+          label = category.category_name
           self.assertIn(label, _ALLOW_LIST,
                         f'Label {label} found but not in label allow list')
 
@@ -297,8 +290,8 @@ class ImageClassifierTest(parameterized.TestCase):
       classifications = image_result.classifications
 
       for classification in classifications:
-        for entry in classification.entries:
-          label = entry.categories[0].category_name
+        for category in classification.categories:
+          label = category.category_name
           self.assertNotIn(label, _DENY_LIST,
                            f'Label {label} found but in deny list.')
 
@@ -324,7 +317,7 @@ class ImageClassifierTest(parameterized.TestCase):
     with _ImageClassifier.create_from_options(options) as classifier:
       # Performs image classification on the input.
       image_result = classifier.classify(self.test_image)
-      self.assertEmpty(image_result.classifications[0].entries[0].categories)
+      self.assertEmpty(image_result.classifications[0].categories)
 
   def test_missing_result_callback(self):
     options = _ImageClassifierOptions(
@@ -402,9 +395,8 @@ class ImageClassifierTest(parameterized.TestCase):
       for timestamp in range(0, 300, 30):
         classification_result = classifier.classify_for_video(
             self.test_image, timestamp)
-        test_utils.assert_proto_equals(
-            self, classification_result.to_pb2(),
-            _generate_burger_results(timestamp).to_pb2())
+        test_utils.assert_proto_equals(self, classification_result.to_pb2(),
+                                       _generate_burger_results().to_pb2())
 
   def test_classify_for_video_succeeds_with_region_of_interest(self):
     custom_classifier_options = _ClassifierOptions(max_results=1)
@@ -423,9 +415,8 @@ class ImageClassifierTest(parameterized.TestCase):
       for timestamp in range(0, 300, 30):
         classification_result = classifier.classify_for_video(
             test_image, timestamp, image_processing_options)
-        test_utils.assert_proto_equals(
-            self, classification_result.to_pb2(),
-            _generate_soccer_ball_results(timestamp).to_pb2())
+        test_utils.assert_proto_equals(self, classification_result.to_pb2(),
+                                       _generate_soccer_ball_results().to_pb2())
 
   def test_calling_classify_in_live_stream_mode(self):
     options = _ImageClassifierOptions(
@@ -460,15 +451,15 @@ class ImageClassifierTest(parameterized.TestCase):
           ValueError, r'Input timestamp must be monotonically increasing'):
         classifier.classify_async(self.test_image, 0)
 
-  @parameterized.parameters((0, _generate_burger_results),
-                            (1, _generate_empty_results))
-  def test_classify_async_calls(self, threshold, expected_result_fn):
+  @parameterized.parameters((0, _generate_burger_results()),
+                            (1, _generate_empty_results()))
+  def test_classify_async_calls(self, threshold, expected_result):
     observed_timestamp_ms = -1
 
-    def check_result(result: _ClassificationResult, output_image: _Image,
+    def check_result(result: ImageClassifierResult, output_image: _Image,
                      timestamp_ms: int):
       test_utils.assert_proto_equals(self, result.to_pb2(),
-                                     expected_result_fn(timestamp_ms).to_pb2())
+                                     expected_result.to_pb2())
       self.assertTrue(
           np.array_equal(output_image.numpy_view(),
                          self.test_image.numpy_view()))
@@ -496,11 +487,10 @@ class ImageClassifierTest(parameterized.TestCase):
     image_processing_options = _ImageProcessingOptions(roi)
     observed_timestamp_ms = -1
 
-    def check_result(result: _ClassificationResult, output_image: _Image,
+    def check_result(result: ImageClassifierResult, output_image: _Image,
                      timestamp_ms: int):
-      test_utils.assert_proto_equals(
-          self, result.to_pb2(),
-          _generate_soccer_ball_results(timestamp_ms).to_pb2())
+      test_utils.assert_proto_equals(self, result.to_pb2(),
+                                     _generate_soccer_ball_results().to_pb2())
       self.assertEqual(output_image.width, test_image.width)
       self.assertEqual(output_image.height, test_image.height)
       self.assertLess(observed_timestamp_ms, timestamp_ms)

mediapipe/tasks/python/text/BUILD

@@ -28,7 +28,7 @@ py_library(
         "//mediapipe/python:packet_getter",
         "//mediapipe/tasks/cc/components/containers/proto:classifications_py_pb2",
         "//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_py_pb2",
-        "//mediapipe/tasks/python/components/containers:classifications",
+        "//mediapipe/tasks/python/components/containers:classification_result",
         "//mediapipe/tasks/python/components/processors:classifier_options",
         "//mediapipe/tasks/python/core:base_options",
         "//mediapipe/tasks/python/core:optional_dependencies",

mediapipe/tasks/python/text/text_classifier.py

@@ -19,21 +19,21 @@ from mediapipe.python import packet_creator
 from mediapipe.python import packet_getter
 from mediapipe.tasks.cc.components.containers.proto import classifications_pb2
 from mediapipe.tasks.cc.text.text_classifier.proto import text_classifier_graph_options_pb2
-from mediapipe.tasks.python.components.containers import classifications
+from mediapipe.tasks.python.components.containers import classification_result as classification_result_module
 from mediapipe.tasks.python.components.processors import classifier_options
 from mediapipe.tasks.python.core import base_options as base_options_module
 from mediapipe.tasks.python.core import task_info as task_info_module
 from mediapipe.tasks.python.core.optional_dependencies import doc_controls
 from mediapipe.tasks.python.text.core import base_text_task_api
 
-TextClassifierResult = classifications.ClassificationResult
+TextClassifierResult = classification_result_module.ClassificationResult
 _BaseOptions = base_options_module.BaseOptions
 _TextClassifierGraphOptionsProto = text_classifier_graph_options_pb2.TextClassifierGraphOptions
 _ClassifierOptions = classifier_options.ClassifierOptions
 _TaskInfo = task_info_module.TaskInfo
 
-_CLASSIFICATION_RESULT_OUT_STREAM_NAME = 'classification_result_out'
-_CLASSIFICATION_RESULT_TAG = 'CLASSIFICATION_RESULT'
+_CLASSIFICATIONS_STREAM_NAME = 'classifications_out'
+_CLASSIFICATIONS_TAG = 'CLASSIFICATIONS'
 _TEXT_IN_STREAM_NAME = 'text_in'
 _TEXT_TAG = 'TEXT'
 _TASK_GRAPH_NAME = 'mediapipe.tasks.text.text_classifier.TextClassifierGraph'
@@ -104,10 +104,7 @@ class TextClassifier(base_text_task_api.BaseTextTaskApi):
         task_graph=_TASK_GRAPH_NAME,
         input_streams=[':'.join([_TEXT_TAG, _TEXT_IN_STREAM_NAME])],
         output_streams=[
-            ':'.join([
-                _CLASSIFICATION_RESULT_TAG,
-                _CLASSIFICATION_RESULT_OUT_STREAM_NAME
-            ])
+            ':'.join([_CLASSIFICATIONS_TAG, _CLASSIFICATIONS_STREAM_NAME])
         ],
         task_options=options)
     return cls(task_info.generate_graph_config())
@@ -131,10 +128,6 @@ class TextClassifier(base_text_task_api.BaseTextTaskApi):
 
     classification_result_proto = classifications_pb2.ClassificationResult()
     classification_result_proto.CopyFrom(
-        packet_getter.get_proto(
-            output_packets[_CLASSIFICATION_RESULT_OUT_STREAM_NAME]))
+        packet_getter.get_proto(output_packets[_CLASSIFICATIONS_STREAM_NAME]))
 
-    return TextClassifierResult([
-        classifications.Classifications.create_from_pb2(classification)
-        for classification in classification_result_proto.classifications
-    ])
+    return TextClassifierResult.create_from_pb2(classification_result_proto)

mediapipe/tasks/python/vision/BUILD

@@ -48,7 +48,7 @@ py_library(
         "//mediapipe/python:packet_getter",
         "//mediapipe/tasks/cc/components/containers/proto:classifications_py_pb2",
         "//mediapipe/tasks/cc/vision/image_classifier/proto:image_classifier_graph_options_py_pb2",
-        "//mediapipe/tasks/python/components/containers:classifications",
+        "//mediapipe/tasks/python/components/containers:classification_result",
         "//mediapipe/tasks/python/components/containers:rect",
         "//mediapipe/tasks/python/components/processors:classifier_options",
         "//mediapipe/tasks/python/core:base_options",

mediapipe/tasks/python/vision/gesture_recognizer.py

@@ -80,7 +80,7 @@ class GestureRecognizerResult:
 def _build_recognition_result(
     output_packets: Mapping[str,
                             packet_module.Packet]) -> GestureRecognizerResult:
-  """Consturcts a `GestureRecognizerResult` from output packets."""
+  """Constructs a `GestureRecognizerResult` from output packets."""
   gestures_proto_list = packet_getter.get_proto_list(
       output_packets[_HAND_GESTURE_STREAM_NAME])
   handedness_proto_list = packet_getter.get_proto_list(
@@ -270,9 +270,9 @@ class GestureRecognizer(base_vision_task_api.BaseVisionTaskApi):
             empty_packet.timestamp.value // _MICRO_SECONDS_PER_MILLISECOND)
         return
 
-      gesture_recognition_result = _build_recognition_result(output_packets)
+      gesture_recognizer_result = _build_recognition_result(output_packets)
       timestamp = output_packets[_HAND_GESTURE_STREAM_NAME].timestamp
-      options.result_callback(gesture_recognition_result, image,
+      options.result_callback(gesture_recognizer_result, image,
                               timestamp.value // _MICRO_SECONDS_PER_MILLISECOND)
 
     task_info = _TaskInfo(

mediapipe/tasks/python/vision/image_classifier.py

@@ -18,12 +18,11 @@ from typing import Callable, Mapping, Optional
 
 from mediapipe.python import packet_creator
 from mediapipe.python import packet_getter
-# TODO: Import MPImage directly one we have an alias
 from mediapipe.python._framework_bindings import image as image_module
 from mediapipe.python._framework_bindings import packet
 from mediapipe.tasks.cc.components.containers.proto import classifications_pb2
 from mediapipe.tasks.cc.vision.image_classifier.proto import image_classifier_graph_options_pb2
-from mediapipe.tasks.python.components.containers import classifications
+from mediapipe.tasks.python.components.containers import classification_result as classification_result_module
 from mediapipe.tasks.python.components.containers import rect
 from mediapipe.tasks.python.components.processors import classifier_options
 from mediapipe.tasks.python.core import base_options as base_options_module
@@ -33,6 +32,7 @@ from mediapipe.tasks.python.vision.core import base_vision_task_api
 from mediapipe.tasks.python.vision.core import image_processing_options as image_processing_options_module
 from mediapipe.tasks.python.vision.core import vision_task_running_mode
 
+ImageClassifierResult = classification_result_module.ClassificationResult
 _NormalizedRect = rect.NormalizedRect
 _BaseOptions = base_options_module.BaseOptions
 _ImageClassifierGraphOptionsProto = image_classifier_graph_options_pb2.ImageClassifierGraphOptions
@@ -41,8 +41,8 @@ _RunningMode = vision_task_running_mode.VisionTaskRunningMode
 _ImageProcessingOptions = image_processing_options_module.ImageProcessingOptions
 _TaskInfo = task_info_module.TaskInfo
 
-_CLASSIFICATION_RESULT_OUT_STREAM_NAME = 'classification_result_out'
-_CLASSIFICATION_RESULT_TAG = 'CLASSIFICATION_RESULT'
+_CLASSIFICATIONS_STREAM_NAME = 'classifications_out'
+_CLASSIFICATIONS_TAG = 'CLASSIFICATIONS'
 _IMAGE_IN_STREAM_NAME = 'image_in'
 _IMAGE_OUT_STREAM_NAME = 'image_out'
 _IMAGE_TAG = 'IMAGE'
@@ -71,9 +71,8 @@ class ImageClassifierOptions:
   base_options: _BaseOptions
   running_mode: _RunningMode = _RunningMode.IMAGE
   classifier_options: _ClassifierOptions = _ClassifierOptions()
-  result_callback: Optional[
-      Callable[[classifications.ClassificationResult, image_module.Image, int],
-               None]] = None
+  result_callback: Optional[Callable[
+      [ImageClassifierResult, image_module.Image, int], None]] = None
 
   @doc_controls.do_not_generate_docs
   def to_pb2(self) -> _ImageClassifierGraphOptionsProto:
@@ -137,17 +136,12 @@ class ImageClassifier(base_vision_task_api.BaseVisionTaskApi):
 
       classification_result_proto = classifications_pb2.ClassificationResult()
       classification_result_proto.CopyFrom(
-          packet_getter.get_proto(
-              output_packets[_CLASSIFICATION_RESULT_OUT_STREAM_NAME]))
-
-      classification_result = classifications.ClassificationResult([
-          classifications.Classifications.create_from_pb2(classification)
-          for classification in classification_result_proto.classifications
-      ])
+          packet_getter.get_proto(output_packets[_CLASSIFICATIONS_STREAM_NAME]))
       image = packet_getter.get_image(output_packets[_IMAGE_OUT_STREAM_NAME])
       timestamp = output_packets[_IMAGE_OUT_STREAM_NAME].timestamp
-      options.result_callback(classification_result, image,
-                              timestamp.value // _MICRO_SECONDS_PER_MILLISECOND)
+      options.result_callback(
+          ImageClassifierResult.create_from_pb2(classification_result_proto),
+          image, timestamp.value // _MICRO_SECONDS_PER_MILLISECOND)
 
     task_info = _TaskInfo(
         task_graph=_TASK_GRAPH_NAME,
@@ -156,10 +150,8 @@ class ImageClassifier(base_vision_task_api.BaseVisionTaskApi):
             ':'.join([_NORM_RECT_TAG, _NORM_RECT_STREAM_NAME]),
         ],
         output_streams=[
-            ':'.join([
-                _CLASSIFICATION_RESULT_TAG,
-                _CLASSIFICATION_RESULT_OUT_STREAM_NAME
-            ]), ':'.join([_IMAGE_TAG, _IMAGE_OUT_STREAM_NAME])
+            ':'.join([_CLASSIFICATIONS_TAG, _CLASSIFICATIONS_STREAM_NAME]),
+            ':'.join([_IMAGE_TAG, _IMAGE_OUT_STREAM_NAME])
         ],
         task_options=options)
     return cls(
@@ -172,7 +164,7 @@ class ImageClassifier(base_vision_task_api.BaseVisionTaskApi):
       self,
       image: image_module.Image,
       image_processing_options: Optional[_ImageProcessingOptions] = None
-  ) -> classifications.ClassificationResult:
+  ) -> ImageClassifierResult:
     """Performs image classification on the provided MediaPipe Image.
 
     Args:
@@ -196,20 +188,16 @@ class ImageClassifier(base_vision_task_api.BaseVisionTaskApi):
 
     classification_result_proto = classifications_pb2.ClassificationResult()
     classification_result_proto.CopyFrom(
-        packet_getter.get_proto(
-            output_packets[_CLASSIFICATION_RESULT_OUT_STREAM_NAME]))
+        packet_getter.get_proto(output_packets[_CLASSIFICATIONS_STREAM_NAME]))
 
-    return classifications.ClassificationResult([
-        classifications.Classifications.create_from_pb2(classification)
-        for classification in classification_result_proto.classifications
-    ])
+    return ImageClassifierResult.create_from_pb2(classification_result_proto)
 
   def classify_for_video(
       self,
       image: image_module.Image,
       timestamp_ms: int,
       image_processing_options: Optional[_ImageProcessingOptions] = None
-  ) -> classifications.ClassificationResult:
+  ) -> ImageClassifierResult:
     """Performs image classification on the provided video frames.
 
     Only use this method when the ImageClassifier is created with the video
@@ -241,13 +229,9 @@ class ImageClassifier(base_vision_task_api.BaseVisionTaskApi):
 
     classification_result_proto = classifications_pb2.ClassificationResult()
     classification_result_proto.CopyFrom(
-        packet_getter.get_proto(
-            output_packets[_CLASSIFICATION_RESULT_OUT_STREAM_NAME]))
+        packet_getter.get_proto(output_packets[_CLASSIFICATIONS_STREAM_NAME]))
 
-    return classifications.ClassificationResult([
-        classifications.Classifications.create_from_pb2(classification)
-        for classification in classification_result_proto.classifications
-    ])
+    return ImageClassifierResult.create_from_pb2(classification_result_proto)
 
   def classify_async(
       self,

mediapipe/tasks/web/components/containers/category.d.ts

@@ -15,7 +15,7 @@
  */
 
 /** A classification category. */
-export interface Category {
+export declare interface Category {
   /** The probability score of this label category. */
   score: number;
 

mediapipe/tasks/web/components/containers/classification_result.d.ts

@@ -17,7 +17,7 @@
 import {Category} from '../../../../tasks/web/components/containers/category';
 
 /** Classification results for a given classifier head. */
-export interface Classifications {
+export declare interface Classifications {
   /**
    * The array of predicted categories, usually sorted by descending scores,
    * e.g., from high to low probability.

mediapipe/tasks/web/components/containers/landmark.d.ts

@@ -20,7 +20,7 @@
  * dimension of image, and the coordinates values are in the range of [0,1].
  * Otherwise, it represenet a point in world coordinates.
  */
-export class Landmark {
+export declare class Landmark {
   /** The x coordinates of the landmark. */
   x: number;
 

mediapipe/tasks/web/components/processors/base_options.ts

@@ -70,17 +70,11 @@ async function configureExternalFile(
 
 /** Configues the `acceleration` option. */
 function configureAcceleration(options: BaseOptions, proto: BaseOptionsProto) {
-  if ('delegate' in options) {
-    const acceleration = new Acceleration();
-    if (options.delegate === 'cpu') {
-      acceleration.setXnnpack(
-          new InferenceCalculatorOptions.Delegate.Xnnpack());
-      proto.setAcceleration(acceleration);
-    } else if (options.delegate === 'gpu') {
-      acceleration.setGpu(new InferenceCalculatorOptions.Delegate.Gpu());
-      proto.setAcceleration(acceleration);
-    } else {
-      proto.clearAcceleration();
-    }
+  const acceleration = proto.getAcceleration() ?? new Acceleration();
+  if (options.delegate === 'gpu') {
+    acceleration.setGpu(new InferenceCalculatorOptions.Delegate.Gpu());
+  } else {
+    acceleration.setTflite(new InferenceCalculatorOptions.Delegate.TfLite());
   }
+  proto.setAcceleration(acceleration);
 }

mediapipe/tasks/web/core/base_options.d.ts

@@ -17,7 +17,7 @@
 // Placeholder for internal dependency on trusted resource url
 
 /** Options to configure MediaPipe Tasks in general. */
-export interface BaseOptions {
+export declare interface BaseOptions {
   /**
    * The model path to the model asset file. Only one of `modelAssetPath` or
    * `modelAssetBuffer` can be set.

mediapipe/tasks/web/core/classifier_options.d.ts

@@ -17,7 +17,7 @@
 import {BaseOptions} from '../../../tasks/web/core/base_options';
 
 /** Options to configure the Mediapipe Classifier Task. */
-export interface ClassifierOptions {
+export declare interface ClassifierOptions {
   /** Options to configure the loading of the model assets. */
   baseOptions?: BaseOptions;
 

mediapipe/tasks/web/core/wasm_loader_options.d.ts

@@ -17,7 +17,7 @@
 // Placeholder for internal dependency on trusted resource url
 
 /** An object containing the locations of all Wasm assets */
-export interface WasmLoaderOptions {
+export declare interface WasmLoaderOptions {
   /** The path to the Wasm loader script. */
   wasmLoaderPath: string;
   /** The path to the Wasm binary. */

mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts

@@ -35,7 +35,7 @@ import {createMediaPipeLib, FileLocator, ImageSource, WasmModule} from '../../..
 // Placeholder for internal dependency on trusted resource url
 
 import {GestureRecognizerOptions} from './gesture_recognizer_options';
-import {GestureRecognitionResult} from './gesture_recognizer_result';
+import {GestureRecognizerResult} from './gesture_recognizer_result';
 
 export {ImageSource};
 
@@ -237,7 +237,7 @@ export class GestureRecognizer extends TaskRunner {
    * @return The detected gestures.
    */
   recognize(imageSource: ImageSource, timestamp: number = performance.now()):
-      GestureRecognitionResult {
+      GestureRecognizerResult {
     this.gestures = [];
     this.landmarks = [];
     this.worldLandmarks = [];

mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer_options.ts

@@ -18,7 +18,7 @@ import {BaseOptions} from '../../../../tasks/web/core/base_options';
 import {ClassifierOptions} from '../../../../tasks/web/core/classifier_options';
 
 /** Options to configure the MediaPipe Gesture Recognizer Task */
-export interface GestureRecognizerOptions {
+export declare interface GestureRecognizerOptions {
   /** Options to configure the loading of the model assets. */
   baseOptions?: BaseOptions;
 

mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer_result.ts

@@ -20,7 +20,7 @@ import {Landmark} from '../../../../tasks/web/components/containers/landmark';
 /**
  * Represents the gesture recognition results generated by `GestureRecognizer`.
  */
-export interface GestureRecognitionResult {
+export declare interface GestureRecognizerResult {
   /** Hand landmarks of detected hands. */
   landmarks: Landmark[][];
 

mediapipe/tasks/web/vision/object_detector/object_detector_result.ts

@@ -17,7 +17,7 @@
 import {Category} from '../../../../tasks/web/components/containers/category';
 
 /** An integer bounding box, axis aligned. */
-export interface BoundingBox {
+export declare interface BoundingBox {
   /** The X coordinate of the top-left corner, in pixels. */
   originX: number;
   /** The Y coordinate of the top-left corner, in pixels. */
@@ -29,7 +29,7 @@ export interface BoundingBox {
 }
 
 /** Represents one object detected by the `ObjectDetector`. */
-export interface Detection {
+export declare interface Detection {
   /** A list of `Category` objects. */
   categories: Category[];