builder()
+ .setTaskName(InteractiveSegmenter.class.getSimpleName())
+ .setTaskRunningModeName(RunningMode.IMAGE.name())
+ .setTaskGraphName(TASK_GRAPH_NAME)
+ .setInputStreams(INPUT_STREAMS)
+ .setOutputStreams(OUTPUT_STREAMS)
+ .setTaskOptions(segmenterOptions)
+ .setEnableFlowLimiting(false)
+ .build(),
+ handler);
+ return new InteractiveSegmenter(runner, segmenterOptions.resultListener().isPresent());
+ }
+
+ /**
+ * Constructor to initialize an {@link InteractiveSegmenter} from a {@link TaskRunner}.
+ *
+ * @param taskRunner a {@link TaskRunner}.
+ */
+ private InteractiveSegmenter(TaskRunner taskRunner, boolean hasResultListener) {
+ super(taskRunner, RunningMode.IMAGE, IMAGE_IN_STREAM_NAME, NORM_RECT_IN_STREAM_NAME);
+ this.hasResultListener = hasResultListener;
+ populateLabels();
+ }
+
+ /**
+ * Populate the labelmap in TensorsToSegmentationCalculator to labels field.
+ *
+ * @throws MediaPipeException if there is an error during finding TensorsToSegmentationCalculator.
+ */
+ private void populateLabels() {
+ CalculatorGraphConfig graphConfig = this.runner.getCalculatorGraphConfig();
+
+ boolean foundTensorsToSegmentation = false;
+ for (CalculatorGraphConfig.Node node : graphConfig.getNodeList()) {
+ if (node.getName().contains(TENSORS_TO_SEGMENTATION_CALCULATOR_NAME)) {
+ if (foundTensorsToSegmentation) {
+ throw new MediaPipeException(
+ MediaPipeException.StatusCode.INTERNAL.ordinal(),
+ "The graph has more than one mediapipe.tasks.TensorsToSegmentationCalculator.");
+ }
+ foundTensorsToSegmentation = true;
+ TensorsToSegmentationCalculatorOptionsProto.TensorsToSegmentationCalculatorOptions options =
+ node.getOptions()
+ .getExtension(
+ TensorsToSegmentationCalculatorOptionsProto
+ .TensorsToSegmentationCalculatorOptions.ext);
+ for (int i = 0; i < options.getLabelItemsMap().size(); i++) {
+ Long labelKey = Long.valueOf(i);
+ if (!options.getLabelItemsMap().containsKey(labelKey)) {
+ throw new MediaPipeException(
+ MediaPipeException.StatusCode.INTERNAL.ordinal(),
+ "The lablemap have no expected key: " + labelKey);
+ }
+ labels.add(options.getLabelItemsMap().get(labelKey).getName());
+ }
+ }
+ }
+ }
+
+ /**
+ * Performs segmentation on the provided single image with default image processing options, given
+ * user's region-of-interest, i.e. without any rotation applied. TODO update java doc
+ * for input image format.
+ *
+ * Users can represent user interaction through {@link RegionOfInterest}, which gives a hint to
+ * perform segmentation focusing on the given region of interest.
+ *
+ *
{@link InteractiveSegmenter} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param roi a {@link RegionOfInterest} object to represent user interaction.
+ * @throws MediaPipeException if there is an internal error. Or if {@link InteractiveSegmenter} is
+ * created with a {@link ResultListener}.
+ */
+ public ImageSegmenterResult segment(MPImage image, RegionOfInterest roi) {
+ return segment(image, roi, ImageProcessingOptions.builder().build());
+ }
+
+ /**
+ * Performs segmentation on the provided single image, given user's region-of-interest.
+ * TODO update java doc for input image format.
+ *
+ * Users can represent user interaction through {@link RegionOfInterest}, which gives a hint to
+ * perform segmentation focusing on the given region of interest.
+ *
+ *
{@link InteractiveSegmenter} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param roi a {@link RegionOfInterest} object to represent user interaction.
+ * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
+ * input image before running inference. Note that region-of-interest is not supported
+ * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
+ * this method throwing an IllegalArgumentException.
+ * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
+ * region-of-interest.
+ * @throws MediaPipeException if there is an internal error. Or if {@link InteractiveSegmenter} is
+ * created with a {@link ResultListener}.
+ */
+ public ImageSegmenterResult segment(
+ MPImage image, RegionOfInterest roi, ImageProcessingOptions imageProcessingOptions) {
+ if (hasResultListener) {
+ throw new MediaPipeException(
+ MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+ "ResultListener is provided in the InteractiveSegmenterOptions, but this method will"
+ + " return an ImageSegmentationResult.");
+ }
+ validateImageProcessingOptions(imageProcessingOptions);
+ return processImageWithRoi(image, roi, imageProcessingOptions);
+ }
+
+ /**
+ * Performs segmentation on the provided single image with default image processing options, given
+ * user's region-of-interest, i.e. without any rotation applied, and provides zero-copied results
+ * via {@link ResultListener} in {@link InteractiveSegmenterOptions}.
+ *
+ * TODO update java doc for input image format.
+ *
+ *
Users can represent user interaction through {@link RegionOfInterest}, which gives a hint to
+ * perform segmentation focusing on the given region of interest.
+ *
+ *
{@link InteractiveSegmenter} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param roi a {@link RegionOfInterest} object to represent user interaction.
+ * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
+ * region-of-interest.
+ * @throws MediaPipeException if there is an internal error. Or if {@link InteractiveSegmenter} is
+ * not created wtih {@link ResultListener} set in {@link InteractiveSegmenterOptions}.
+ */
+ public void segmentWithResultListener(MPImage image, RegionOfInterest roi) {
+ segmentWithResultListener(image, roi, ImageProcessingOptions.builder().build());
+ }
+
+ /**
+ * Performs segmentation on the provided single image given user's region-of-interest, and
+ * provides zero-copied results via {@link ResultListener} in {@link InteractiveSegmenterOptions}.
+ *
+ * TODO update java doc for input image format.
+ *
+ *
Users can represent user interaction through {@link RegionOfInterest}, which gives a hint to
+ * perform segmentation focusing on the given region of interest.
+ *
+ *
{@link InteractiveSegmenter} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param roi a {@link RegionOfInterest} object to represent user interaction.
+ * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
+ * input image before running inference. Note that region-of-interest is not supported
+ * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
+ * this method throwing an IllegalArgumentException.
+ * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
+ * region-of-interest.
+ * @throws MediaPipeException if there is an internal error. Or if {@link InteractiveSegmenter} is
+ * not created wtih {@link ResultListener} set in {@link InteractiveSegmenterOptions}.
+ */
+ public void segmentWithResultListener(
+ MPImage image, RegionOfInterest roi, ImageProcessingOptions imageProcessingOptions) {
+ if (!hasResultListener) {
+ throw new MediaPipeException(
+ MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+ "ResultListener is not set in the InteractiveSegmenterOptions, but this method expects a"
+ + " ResultListener to process ImageSegmentationResult.");
+ }
+ validateImageProcessingOptions(imageProcessingOptions);
+ ImageSegmenterResult unused = processImageWithRoi(image, roi, imageProcessingOptions);
+ }
+
+ /**
+ * Get the category label list of the ImageSegmenter can recognize. For CATEGORY_MASK type, the
+ * index in the category mask corresponds to the category in the label list. For CONFIDENCE_MASK
+ * type, the output mask list at index corresponds to the category in the label list.
+ *
+ * If there is no labelmap provided in the model file, empty label list is returned.
+ */
+ List getLabels() {
+ return labels;
+ }
+
+ /** Options for setting up an {@link InteractiveSegmenter}. */
+ @AutoValue
+ public abstract static class InteractiveSegmenterOptions extends TaskOptions {
+
+ /** Builder for {@link InteractiveSegmenterOptions}. */
+ @AutoValue.Builder
+ public abstract static class Builder {
+ /** Sets the base options for the image segmenter task. */
+ public abstract Builder setBaseOptions(BaseOptions value);
+
+ /** The output type from image segmenter. */
+ public abstract Builder setOutputType(OutputType value);
+
+ /**
+ * Sets an optional {@link ResultListener} to receive the segmentation results when the graph
+ * pipeline is done processing an image.
+ */
+ public abstract Builder setResultListener(
+ ResultListener value);
+
+ /** Sets an optional {@link ErrorListener}}. */
+ public abstract Builder setErrorListener(ErrorListener value);
+
+ abstract InteractiveSegmenterOptions autoBuild();
+
+ /** Builds the {@link InteractiveSegmenterOptions} instance. */
+ public final InteractiveSegmenterOptions build() {
+ return autoBuild();
+ }
+ }
+
+ abstract BaseOptions baseOptions();
+
+ abstract OutputType outputType();
+
+ abstract Optional> resultListener();
+
+ abstract Optional errorListener();
+
+ /** The output type of segmentation results. */
+ public enum OutputType {
+ // Gives a single output mask where each pixel represents the class which
+ // the pixel in the original image was predicted to belong to.
+ CATEGORY_MASK,
+ // Gives a list of output masks where, for each mask, each pixel represents
+ // the prediction confidence, usually in the [0, 1] range.
+ CONFIDENCE_MASK
+ }
+
+ public static Builder builder() {
+ return new AutoValue_InteractiveSegmenter_InteractiveSegmenterOptions.Builder()
+ .setOutputType(OutputType.CATEGORY_MASK);
+ }
+
+ /**
+ * Converts an {@link InteractiveSegmenterOptions} to a {@link CalculatorOptions} protobuf
+ * message.
+ */
+ @Override
+ public CalculatorOptions convertToCalculatorOptionsProto() {
+ ImageSegmenterGraphOptionsProto.ImageSegmenterGraphOptions.Builder taskOptionsBuilder =
+ ImageSegmenterGraphOptionsProto.ImageSegmenterGraphOptions.newBuilder()
+ .setBaseOptions(
+ BaseOptionsProto.BaseOptions.newBuilder()
+ .setUseStreamMode(false)
+ .mergeFrom(convertBaseOptionsToProto(baseOptions()))
+ .build());
+
+ SegmenterOptionsProto.SegmenterOptions.Builder segmenterOptionsBuilder =
+ SegmenterOptionsProto.SegmenterOptions.newBuilder();
+ if (outputType() == OutputType.CONFIDENCE_MASK) {
+ segmenterOptionsBuilder.setOutputType(
+ SegmenterOptionsProto.SegmenterOptions.OutputType.CONFIDENCE_MASK);
+ } else if (outputType() == OutputType.CATEGORY_MASK) {
+ segmenterOptionsBuilder.setOutputType(
+ SegmenterOptionsProto.SegmenterOptions.OutputType.CATEGORY_MASK);
+ }
+
+ taskOptionsBuilder.setSegmenterOptions(segmenterOptionsBuilder);
+ return CalculatorOptions.newBuilder()
+ .setExtension(
+ ImageSegmenterGraphOptionsProto.ImageSegmenterGraphOptions.ext,
+ taskOptionsBuilder.build())
+ .build();
+ }
+ }
+
+ /**
+ * Validates that the provided {@link ImageProcessingOptions} doesn't contain a
+ * region-of-interest.
+ */
+ private static void validateImageProcessingOptions(
+ ImageProcessingOptions imageProcessingOptions) {
+ if (imageProcessingOptions.regionOfInterest().isPresent()) {
+ throw new IllegalArgumentException(
+ "InteractiveSegmenter doesn't support region-of-interest.");
+ }
+ }
+
+ /** The Region-Of-Interest (ROI) to interact with. */
+ public static class RegionOfInterest {
+ private NormalizedKeypoint keypoint;
+
+ private RegionOfInterest() {}
+
+ /**
+ * Creates a {@link RegionOfInterest} instance representing a single point pointing to the
+ * object that the user wants to segment.
+ */
+ public static RegionOfInterest create(NormalizedKeypoint keypoint) {
+ RegionOfInterest roi = new RegionOfInterest();
+ roi.keypoint = keypoint;
+ return roi;
+ }
+ }
+
+ /**
+ * Converts a {@link RegionOfInterest} instance into a {@link RenderData} protobuf message
+ *
+ * @param roi a {@link RegionOfInterest} object to represent user interaction.
+ * @throws IllegalArgumentException if {@link RegionOfInterest} does not represent a valid user
+ * interaction.
+ */
+ private static RenderData convertToRenderData(RegionOfInterest roi) {
+ RenderData.Builder builder = RenderData.newBuilder();
+ if (roi.keypoint != null) {
+ return builder
+ .addRenderAnnotations(
+ RenderAnnotation.newBuilder()
+ .setColor(Color.newBuilder().setR(255))
+ .setPoint(
+ RenderAnnotation.Point.newBuilder()
+ .setX(roi.keypoint.x())
+ .setY(roi.keypoint.y())))
+ .build();
+ }
+
+ throw new IllegalArgumentException(
+ "RegionOfInterest does not include a valid user interaction");
+ }
+
+ /**
+ * A synchronous method to process single image inputs. The call blocks the current thread until a
+ * failure status or a successful result is returned.
+ *
+ * This is almost the same as {@link BaseVisionTaskApi.processImageData} except accepting an
+ * additional {@link RegionOfInterest}.
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param roi a {@link RegionOfInterest} object to represent user interaction.
+ * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
+ * input image before running inference.
+ * @throws MediaPipeException if the task is not in the image mode.
+ */
+ private ImageSegmenterResult processImageWithRoi(
+ MPImage image, RegionOfInterest roi, ImageProcessingOptions imageProcessingOptions) {
+ if (runningMode != RunningMode.IMAGE) {
+ throw new MediaPipeException(
+ MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+ "Task is not initialized with the image mode. Current running mode:"
+ + runningMode.name());
+ }
+ Map inputPackets = new HashMap<>();
+ inputPackets.put(IMAGE_IN_STREAM_NAME, runner.getPacketCreator().createImage(image));
+ RenderData renderData = convertToRenderData(roi);
+ inputPackets.put(ROI_IN_STREAM_NAME, runner.getPacketCreator().createProto(renderData));
+ inputPackets.put(
+ NORM_RECT_IN_STREAM_NAME,
+ runner.getPacketCreator().createProto(convertToNormalizedRect(imageProcessingOptions)));
+ return (ImageSegmenterResult) runner.process(inputPackets);
+ }
+}
diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/interactivesegmenter/AndroidManifest.xml b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/interactivesegmenter/AndroidManifest.xml
new file mode 100644
index 00000000..97280f5e
--- /dev/null
+++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/interactivesegmenter/AndroidManifest.xml
@@ -0,0 +1,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/interactivesegmenter/BUILD b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/interactivesegmenter/BUILD
new file mode 100644
index 00000000..c1448676
--- /dev/null
+++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/interactivesegmenter/BUILD
@@ -0,0 +1,19 @@
+# Copyright 2023 The MediaPipe Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+# TODO: Enable this in OSS
diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/interactivesegmenter/InteractiveSegmenterTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/interactivesegmenter/InteractiveSegmenterTest.java
new file mode 100644
index 00000000..0d958143
--- /dev/null
+++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/interactivesegmenter/InteractiveSegmenterTest.java
@@ -0,0 +1,92 @@
+// Copyright 2023 The MediaPipe Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.vision.interactivesegmenter;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.res.AssetManager;
+import android.graphics.BitmapFactory;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.mediapipe.framework.image.BitmapImageBuilder;
+import com.google.mediapipe.framework.image.MPImage;
+import com.google.mediapipe.tasks.components.containers.NormalizedKeypoint;
+import com.google.mediapipe.tasks.core.BaseOptions;
+import com.google.mediapipe.tasks.vision.imagesegmenter.ImageSegmenterResult;
+import com.google.mediapipe.tasks.vision.interactivesegmenter.InteractiveSegmenter.InteractiveSegmenterOptions;
+import java.io.InputStream;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+/** Test for {@link InteractiveSegmenter}. */
+@RunWith(Suite.class)
+@SuiteClasses({
+ InteractiveSegmenterTest.General.class,
+})
+public class InteractiveSegmenterTest {
+ private static final String DEEPLAB_MODEL_FILE = "ptm_512_hdt_ptm_woid.tflite";
+ private static final String CATS_AND_DOGS_IMAGE = "cats_and_dogs.jpg";
+ private static final int MAGNIFICATION_FACTOR = 10;
+
+ @RunWith(AndroidJUnit4.class)
+ public static final class General extends InteractiveSegmenterTest {
+ @Test
+ public void segment_successWithCategoryMask() throws Exception {
+ final String inputImageName = CATS_AND_DOGS_IMAGE;
+ final InteractiveSegmenter.RegionOfInterest roi =
+ InteractiveSegmenter.RegionOfInterest.create(NormalizedKeypoint.create(0.25f, 0.9f));
+ InteractiveSegmenterOptions options =
+ InteractiveSegmenterOptions.builder()
+ .setBaseOptions(BaseOptions.builder().setModelAssetPath(DEEPLAB_MODEL_FILE).build())
+ .setOutputType(InteractiveSegmenterOptions.OutputType.CATEGORY_MASK)
+ .build();
+ InteractiveSegmenter imageSegmenter =
+ InteractiveSegmenter.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ MPImage image = getImageFromAsset(inputImageName);
+ ImageSegmenterResult actualResult = imageSegmenter.segment(image, roi);
+ List segmentations = actualResult.segmentations();
+ assertThat(segmentations.size()).isEqualTo(1);
+ }
+
+ @Test
+ public void segment_successWithConfidenceMask() throws Exception {
+ final String inputImageName = CATS_AND_DOGS_IMAGE;
+ final InteractiveSegmenter.RegionOfInterest roi =
+ InteractiveSegmenter.RegionOfInterest.create(NormalizedKeypoint.create(0.25f, 0.9f));
+ InteractiveSegmenterOptions options =
+ InteractiveSegmenterOptions.builder()
+ .setBaseOptions(BaseOptions.builder().setModelAssetPath(DEEPLAB_MODEL_FILE).build())
+ .setOutputType(InteractiveSegmenterOptions.OutputType.CONFIDENCE_MASK)
+ .build();
+ InteractiveSegmenter imageSegmenter =
+ InteractiveSegmenter.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ ImageSegmenterResult actualResult =
+ imageSegmenter.segment(getImageFromAsset(inputImageName), roi);
+ List segmentations = actualResult.segmentations();
+ assertThat(segmentations.size()).isEqualTo(2);
+ }
+ }
+
+ private static MPImage getImageFromAsset(String filePath) throws Exception {
+ AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets();
+ InputStream istr = assetManager.open(filePath);
+ return new BitmapImageBuilder(BitmapFactory.decodeStream(istr)).build();
+ }
+}