builder()
+ .setTaskGraphName(TASK_GRAPH_NAME)
+ .setInputStreams(INPUT_STREAMS)
+ .setOutputStreams(OUTPUT_STREAMS)
+ .setTaskOptions(options)
+ .setEnableFlowLimiting(false)
+ .build(),
+ handler);
+ return new AudioEmbedder(runner, options.runningMode());
+ }
+
+ /**
+ * Constructor to initialize an {@link AudioEmbedder} from a {@link TaskRunner} and {@link
+ * RunningMode}.
+ *
+ * @param taskRunner a {@link TaskRunner}.
+ * @param runningMode a mediapipe audio task {@link RunningMode}.
+ */
+ private AudioEmbedder(TaskRunner taskRunner, RunningMode runningMode) {
+ super(taskRunner, runningMode, AUDIO_IN_STREAM_NAME, SAMPLE_RATE_IN_STREAM_NAME);
+ }
+
+ /*
+ * Performs embedding extraction on the provided audio clips. Only use this method when the
+ * AudioEmbedder is created with the audio clips mode.
+ *
+ * The audio clip is represented as a MediaPipe {@link AudioData} object The method accepts
+ * audio clips with various length and audio sample rate. It's required to provide the
+ * corresponding audio sample rate within the {@link AudioData} object.
+ *
+ *
The input audio clip may be longer than what the model is able to process in a single
+ * inference. When this occurs, the input audio clip is split into multiple chunks starting at
+ * different timestamps. For this reason, this function returns a vector of EmbeddingResult
+ * objects, each associated with a timestamp corresponding to the start (in milliseconds) of the
+ * chunk data that was extracted.
+ *
+ * @param audioClip a MediaPipe {@link AudioData} object for processing.
+ * @throws MediaPipeException if there is an internal error.
+ */
+ public AudioEmbedderResult embed(AudioData audioClip) {
+ return (AudioEmbedderResult) processAudioClip(audioClip);
+ }
+
+ /*
+ * Sends audio data (a block in a continuous audio stream) to perform audio embedding, and
+ * the results will be available via the {@link ResultListener} provided in the
+ * {@link AudioClassifierOptions}. Only use this method when the AudioEmbedder is created with
+ * the audio stream mode.
+ *
+ *
The audio block is represented as a MediaPipe {@link AudioData} object. The audio data will
+ * be resampled, accumulated, and framed to the proper size for the underlying model to consume.
+ * It's required to provide the corresponding audio sample rate within {@link AudioData} object as
+ * well as a timestamp (in milliseconds) to indicate the start time of the input audio block. The
+ * timestamps must be monotonically increasing. This method will return immediately after
+ * the input audio data is accepted. The results will be available in the `resultListener`
+ * provided in the `AudioEmbedderOptions`. The `embedAsync` method is designed to process
+ * auido stream data such as microphone input.
+ *
+ *
The input audio block may be longer than what the model is able to process in a single
+ * inference. When this occurs, the input audio block is split into multiple chunks. For this
+ * reason, the callback may be called multiple times (once per chunk) for each call to this
+ * function.
+ *
+ * @param audioBlock a MediaPipe {@link AudioData} object for processing.
+ * @param timestampMs the input timestamp (in milliseconds).
+ * @throws MediaPipeException if there is an internal error.
+ */
+ public void embedAsync(AudioData audioBlock, long timestampMs) {
+ checkOrSetSampleRate(audioBlock.getFormat().getSampleRate());
+ sendAudioStreamData(audioBlock, timestampMs);
+ }
+
+ /**
+ * Utility function to compute cosine
+ * similarity between two {@link Embedding} objects.
+ *
+ * @throws IllegalArgumentException if the embeddings are of different types (float vs.
+ * quantized), have different sizes, or have an L2-norm of 0.
+ */
+ public static double cosineSimilarity(Embedding u, Embedding v) {
+ return CosineSimilarity.compute(u, v);
+ }
+
+ /** Options for setting up and {@link AudioEmbedder}. */
+ @AutoValue
+ public abstract static class AudioEmbedderOptions extends TaskOptions {
+
+ /** Builder for {@link AudioEmbedderOptions}. */
+ @AutoValue.Builder
+ public abstract static class Builder {
+ /** Sets the {@link BaseOptions} for the audio embedder task. */
+ public abstract Builder setBaseOptions(BaseOptions baseOptions);
+
+ /**
+ * Sets the {@link RunningMode} for the audio embedder task. Default to the audio clips mode.
+ * Image embedder has two modes:
+ *
+ *
+ * - AUDIO_CLIPS: The mode for running audio embedding on audio clips. Users feed audio
+ * clips to the `embed` method, and will receive the embedding results as the return
+ * value.
+ *
- AUDIO_STREAM: The mode for running audio embedding on the audio stream, such as from
+ * microphone. Users call `embedAsync` to push the audio data into the AudioEmbedder,
+ * the embedding results will be available in the result callback when the audio
+ * embedder finishes the work.
+ *
+ */
+ public abstract Builder setRunningMode(RunningMode runningMode);
+
+ /**
+ * Sets the optional {@link EmbedderOptions} controling embedding behavior, such as score
+ * threshold, number of results, etc.
+ */
+ public abstract Builder setEmbedderOptions(EmbedderOptions embedderOptions);
+
+ /**
+ * Sets the {@link ResultListener} to receive the embedding results asynchronously when the
+ * audio embedder is in the audio stream mode.
+ */
+ public abstract Builder setResultListener(
+ PureResultListener resultListener);
+
+ /** Sets an optional {@link ErrorListener}. */
+ public abstract Builder setErrorListener(ErrorListener errorListener);
+
+ abstract AudioEmbedderOptions autoBuild();
+
+ /**
+ * Validates and builds the {@link AudioEmbedderOptions} instance.
+ *
+ * @throws IllegalArgumentException if the result listener and the running mode are not
+ * properly configured. The result listener should only be set when the audio embedder is
+ * in the audio stream mode.
+ */
+ public final AudioEmbedderOptions build() {
+ AudioEmbedderOptions options = autoBuild();
+ if (options.runningMode() == RunningMode.AUDIO_STREAM) {
+ if (!options.resultListener().isPresent()) {
+ throw new IllegalArgumentException(
+ "The audio embedder is in the audio stream mode, a user-defined result listener"
+ + " must be provided in the AudioEmbedderOptions.");
+ }
+ } else if (options.resultListener().isPresent()) {
+ throw new IllegalArgumentException(
+ "The audio embedder is in the audio clips mode, a user-defined result listener"
+ + " shouldn't be provided in AudioEmbedderOptions.");
+ }
+ return options;
+ }
+ }
+
+ abstract BaseOptions baseOptions();
+
+ abstract RunningMode runningMode();
+
+ abstract Optional embedderOptions();
+
+ abstract Optional> resultListener();
+
+ abstract Optional errorListener();
+
+ public static Builder builder() {
+ return new AutoValue_AudioEmbedder_AudioEmbedderOptions.Builder()
+ .setRunningMode(RunningMode.AUDIO_CLIPS);
+ }
+
+ /** Converts a {@link AudioEmbedderOptions} to a {@link CalculatorOptions} protobuf message. */
+ @Override
+ public CalculatorOptions convertToCalculatorOptionsProto() {
+ BaseOptionsProto.BaseOptions.Builder baseOptionsBuilder =
+ BaseOptionsProto.BaseOptions.newBuilder();
+ baseOptionsBuilder.setUseStreamMode(runningMode() == RunningMode.AUDIO_STREAM);
+ baseOptionsBuilder.mergeFrom(convertBaseOptionsToProto(baseOptions()));
+ AudioEmbedderGraphOptionsProto.AudioEmbedderGraphOptions.Builder taskOptionsBuilder =
+ AudioEmbedderGraphOptionsProto.AudioEmbedderGraphOptions.newBuilder()
+ .setBaseOptions(baseOptionsBuilder);
+ if (embedderOptions().isPresent()) {
+ taskOptionsBuilder.setEmbedderOptions(embedderOptions().get().convertToProto());
+ }
+ return CalculatorOptions.newBuilder()
+ .setExtension(
+ AudioEmbedderGraphOptionsProto.AudioEmbedderGraphOptions.ext,
+ taskOptionsBuilder.build())
+ .build();
+ }
+ }
+}
diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioembedder/AudioEmbedderResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioembedder/AudioEmbedderResult.java
new file mode 100644
index 00000000..ee4df019
--- /dev/null
+++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioembedder/AudioEmbedderResult.java
@@ -0,0 +1,75 @@
+// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.audio.audioembedder;
+
+import com.google.auto.value.AutoValue;
+import com.google.mediapipe.tasks.components.containers.EmbeddingResult;
+import com.google.mediapipe.tasks.components.containers.proto.EmbeddingsProto;
+import com.google.mediapipe.tasks.core.TaskResult;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/** Represents the embedding results generated by {@link AudioEmbedder}. */
+@AutoValue
+public abstract class AudioEmbedderResult implements TaskResult {
+
+ /**
+ * Creates an {@link AudioEmbedderResult} instance from a list of {@link
+ * EmbeddingsProto.EmbeddingResult} protobuf messages.
+ *
+ * @param protoList a list of {@link EmbeddingsProto.EmbeddingResult} protobuf message to convert.
+ * @param timestampMs a timestamp for this result.
+ */
+ static AudioEmbedderResult createFromProtoList(
+ List protoList, long timestampMs) {
+ List classificationResultList = new ArrayList<>();
+ for (EmbeddingsProto.EmbeddingResult proto : protoList) {
+ classificationResultList.add(EmbeddingResult.createFromProto(proto));
+ }
+ return new AutoValue_AudioEmbedderResult(
+ Optional.of(classificationResultList), Optional.empty(), timestampMs);
+ }
+
+ /**
+ * Creates an {@link AudioEmbedderResult} instance from a {@link EmbeddingsProto.EmbeddingResult}
+ * protobuf message.
+ *
+ * @param proto the {@link EmbeddingsProto.EmbeddingResult} protobuf message to convert.
+ * @param timestampMs a timestamp for this result.
+ */
+ static AudioEmbedderResult createFromProto(
+ EmbeddingsProto.EmbeddingResult proto, long timestampMs) {
+ return new AutoValue_AudioEmbedderResult(
+ Optional.empty(), Optional.of(EmbeddingResult.createFromProto(proto)), timestampMs);
+ }
+
+ /**
+ * A list of of timpstamped {@link EmbeddingResult} objects, each contains one set of results per
+ * embedder head. The list represents the audio embedding result of an audio clip, and is only
+ * available when running with the audio clips mode.
+ */
+ public abstract Optional> embeddingResultList();
+
+ /**
+ * Contains one set of results per classifier head. A {@link EmbeddingResult} usually represents
+ * one audio embedding result in an audio stream, and s only available when running with the audio
+ * stream mode.
+ */
+ public abstract Optional embeddingResult();
+
+ @Override
+ public abstract long timestampMs();
+}
diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl
index ab7ad661..358dd8d1 100644
--- a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl
+++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl
@@ -32,6 +32,7 @@ _CORE_TASKS_JAVA_PROTO_LITE_TARGETS = [
_AUDIO_TASKS_JAVA_PROTO_LITE_TARGETS = [
"//mediapipe/tasks/cc/audio/audio_classifier/proto:audio_classifier_graph_options_java_proto_lite",
+ "//mediapipe/tasks/cc/audio/audio_embedder/proto:audio_embedder_graph_options_java_proto_lite",
]
_VISION_TASKS_JAVA_PROTO_LITE_TARGETS = [