From b3d999704f2e4b254670817eaf9217f61939599f Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 29 Mar 2023 08:34:15 -0700 Subject: [PATCH] FaceStylizer Java API PiperOrigin-RevId: 520344417 --- .../mediapipe/tasks/mediapipe_tasks_aar.bzl | 1 + .../com/google/mediapipe/tasks/vision/BUILD | 24 + .../vision/facestylizer/AndroidManifest.xml | 8 + .../vision/facestylizer/FaceStylizer.java | 571 ++++++++++++++++++ .../facestylizer/FaceStylizerResult.java | 39 ++ .../vision/facestylizer/AndroidManifest.xml | 24 + .../mediapipe/tasks/vision/facestylizer/BUILD | 19 + .../vision/facestylizer/FaceStylizerTest.java | 391 ++++++++++++ 8 files changed, 1077 insertions(+) create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/AndroidManifest.xml create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizer.java create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizerResult.java create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/AndroidManifest.xml create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/BUILD create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizerTest.java diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl index 1f614601..641dbd3b 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl @@ -41,6 +41,7 @@ _VISION_TASKS_JAVA_PROTO_LITE_TARGETS = [ "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_stylizer/proto:face_stylizer_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_embedder_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_java_proto_lite", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD index be5f8a24..989c95e9 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD @@ -47,6 +47,7 @@ cc_binary( "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", + "//mediapipe/tasks/cc/vision/face_stylizer:face_stylizer_graph", "//mediapipe/tasks/cc/vision/gesture_recognizer:gesture_recognizer_graph", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/cc/vision/image_embedder:image_embedder_graph", @@ -115,6 +116,29 @@ android_library( ], ) +android_library( + name = "facestylizer", + srcs = [ + "facestylizer/FaceStylizer.java", + "facestylizer/FaceStylizerResult.java", + ], + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], + manifest = "imagesegmenter/AndroidManifest.xml", + deps = [ + ":core", + "//mediapipe/framework:calculator_options_java_proto_lite", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/java/com/google/mediapipe/framework/image", + "//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_stylizer/proto:face_stylizer_graph_options_java_proto_lite", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", + "//third_party:autovalue", + "@maven//:com_google_guava_guava", + ], +) + android_library( name = "gesturerecognizer", srcs = [ diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/AndroidManifest.xml b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/AndroidManifest.xml new file mode 100644 index 00000000..ffed33c9 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + + diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizer.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizer.java new file mode 100644 index 00000000..2c47316f --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizer.java @@ -0,0 +1,571 @@ +// Copyright 2023 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.facestylizer; + +import android.content.Context; +import com.google.auto.value.AutoValue; +import com.google.mediapipe.proto.CalculatorOptionsProto.CalculatorOptions; +import com.google.mediapipe.framework.AndroidPacketGetter; +import com.google.mediapipe.framework.MediaPipeException; +import com.google.mediapipe.framework.Packet; +import com.google.mediapipe.framework.PacketGetter; +import com.google.mediapipe.framework.image.BitmapImageBuilder; +import com.google.mediapipe.framework.image.ByteBufferImageBuilder; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.core.BaseOptions; +import com.google.mediapipe.tasks.core.ErrorListener; +import com.google.mediapipe.tasks.core.OutputHandler; +import com.google.mediapipe.tasks.core.OutputHandler.ResultListener; +import com.google.mediapipe.tasks.core.TaskInfo; +import com.google.mediapipe.tasks.core.TaskOptions; +import com.google.mediapipe.tasks.core.TaskRunner; +import com.google.mediapipe.tasks.core.proto.BaseOptionsProto; +import com.google.mediapipe.tasks.vision.core.BaseVisionTaskApi; +import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions; +import com.google.mediapipe.tasks.vision.core.RunningMode; +import com.google.mediapipe.tasks.vision.facestylizer.proto.FaceStylizerGraphOptionsProto; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** + * Performs face stylization on images. + * + *

Note that, in addition to the standard stylization API, {@link #stylize} and {@link + * #stylizeForVideo}, that take an input image and return the outputs, but involves deep copy of the + * returns, FaceStylizer also supports the callback API, {@link #stylizeWithResultListener} and + * {@link #stylizeForVideoWithResultListener}, which allow you to access the outputs through zero + * copy for the duration of the result listener. + * + *

The callback API is available for all {@link RunningMode} in FaceStylizer. Set {@link + * ResultListener} in {@link FaceStylizerOptions} properly to use the callback API. + * + *

The API expects a TFLite model with,TFLite Model Metadata.. + * + *

+ */ +public final class FaceStylizer extends BaseVisionTaskApi { + private static final String IMAGE_IN_STREAM_NAME = "image_in"; + private static final String NORM_RECT_IN_STREAM_NAME = "norm_rect_in"; + private static final String IMAGE_OUT_STREAM_NAME = "image_out"; + + @SuppressWarnings("ConstantCaseForConstants") + private static final List INPUT_STREAMS = + Collections.unmodifiableList( + Arrays.asList("IMAGE:" + IMAGE_IN_STREAM_NAME, "NORM_RECT:" + NORM_RECT_IN_STREAM_NAME)); + + @SuppressWarnings("ConstantCaseForConstants") + private static final List OUTPUT_STREAMS = + Collections.singletonList("STYLIZED_IMAGE:" + IMAGE_OUT_STREAM_NAME); + + private static final int IMAGE_OUT_STREAM_INDEX = 0; + private static final String TASK_GRAPH_NAME = + "mediapipe.tasks.vision.face_stylizer.FaceStylizerGraph"; + private final boolean hasResultListener; + + /** + * Creates an {@link FaceStylizer} instance from an {@link FaceStylizerOptions}. + * + * @param context an Android {@link Context}. + * @param stylizerOptions an {@link FaceStylizerOptions} instance. + * @throws MediaPipeException if there is an error during {@link FaceStylizer} creation. + */ + public static FaceStylizer createFromOptions( + Context context, FaceStylizerOptions stylizerOptions) { + // TODO: Consolidate OutputHandler and TaskRunner. + OutputHandler handler = new OutputHandler<>(); + handler.setOutputPacketConverter( + new OutputHandler.OutputPacketConverter<>() { + @Override + public FaceStylizerResult convertToTaskResult(List packets) + throws MediaPipeException { + Packet packet = packets.get(IMAGE_OUT_STREAM_INDEX); + int width = PacketGetter.getImageWidth(packet); + int height = PacketGetter.getImageHeight(packet); + int numChannels = PacketGetter.getImageNumChannels(packet); + int imageFormat = + numChannels == 3 ? MPImage.IMAGE_FORMAT_RGB : MPImage.IMAGE_FORMAT_RGBA; + + ByteBuffer imageBuffer; + // If resultListener is not provided, the resulted MPImage is deep copied from the + // MediaPipe graph. If provided, the result MPImage is wrapping the MediaPipe packet + // memory. + if (!stylizerOptions.resultListener().isPresent()) { + imageBuffer = ByteBuffer.allocateDirect(width * height * numChannels); + if (!PacketGetter.getImageData(packet, imageBuffer)) { + imageBuffer = null; + } + } else { + imageBuffer = PacketGetter.getImageDataDirectly(packet); + } + + if (imageBuffer == null) { + throw new MediaPipeException( + MediaPipeException.StatusCode.INTERNAL.ordinal(), + "There is an error getting the stylized face. It usually results from incorrect" + + " options of unsupported OutputType of given model."); + } + ByteBufferImageBuilder imageBuilder = + new ByteBufferImageBuilder(imageBuffer, width, height, imageFormat); + + return FaceStylizerResult.create( + imageBuilder.build(), + BaseVisionTaskApi.generateResultTimestampMs( + stylizerOptions.runningMode(), packets.get(IMAGE_OUT_STREAM_INDEX))); + } + + @Override + public MPImage convertToTaskInput(List packets) { + return new BitmapImageBuilder( + AndroidPacketGetter.getBitmapFromRgb(packets.get(IMAGE_OUT_STREAM_INDEX))) + .build(); + } + }); + stylizerOptions.resultListener().ifPresent(handler::setResultListener); + stylizerOptions.errorListener().ifPresent(handler::setErrorListener); + TaskRunner runner = + TaskRunner.create( + context, + TaskInfo.builder() + .setTaskName(FaceStylizer.class.getSimpleName()) + .setTaskRunningModeName(stylizerOptions.runningMode().name()) + .setTaskGraphName(TASK_GRAPH_NAME) + .setInputStreams(INPUT_STREAMS) + .setOutputStreams(OUTPUT_STREAMS) + .setTaskOptions(stylizerOptions) + .setEnableFlowLimiting(stylizerOptions.runningMode() == RunningMode.LIVE_STREAM) + .build(), + handler); + return new FaceStylizer( + runner, stylizerOptions.runningMode(), stylizerOptions.resultListener().isPresent()); + } + + /** + * Constructor to initialize an {@link FaceStylizer} from a {@link TaskRunner} and a {@link + * RunningMode}. + * + * @param taskRunner a {@link TaskRunner}. + * @param runningMode a mediapipe vision task {@link RunningMode}. + */ + private FaceStylizer(TaskRunner taskRunner, RunningMode runningMode, boolean hasResultListener) { + super(taskRunner, runningMode, IMAGE_IN_STREAM_NAME, NORM_RECT_IN_STREAM_NAME); + this.hasResultListener = hasResultListener; + } + + /** + * Performs face stylization on the provided single image with default image processing options, + * i.e. without any rotation applied. Only use this method when the {@link FaceStylizer} is + * created with {@link RunningMode#IMAGE}. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + *

The image can be of any size. To ensure that the output image has reasonable quality, the + * size of the stylized output is based the model output size and can be smaller than the input + * image. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @throws MediaPipeException if there is an internal error. Or if {@link FaceStylizer} is created + * with a {@link ResultListener}. + */ + public FaceStylizerResult stylize(MPImage image) { + return stylize(image, ImageProcessingOptions.builder().build()); + } + + /** + * Performs face stylization on the provided single image. Only use this method when the {@link + * FaceStylizer} is created with {@link RunningMode#IMAGE}. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + *

The input image can be of any size, To ensure that the output image has reasonable quality, + * the stylized output image size is the smaller of the model output size and the size of the + * {@link ImageProcessingOptions#regionOfInterest} specified in {@code imageProcessingOptions}. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the + * input image before running inference. Note that region-of-interest is not supported + * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in + * this method throwing an IllegalArgumentException. + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + * @throws MediaPipeException if there is an internal error. Or if {@link FaceStylizer} is created + * with a {@link ResultListener}. + */ + public FaceStylizerResult stylize(MPImage image, ImageProcessingOptions imageProcessingOptions) { + if (hasResultListener) { + throw new MediaPipeException( + MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), + "ResultListener is provided in the FaceStylizerOptions, but this method will return an" + + " ImageSegmentationResult."); + } + return (FaceStylizerResult) processImageData(image, imageProcessingOptions); + } + + /** + * Performs face stylization on the provided single image with default image processing options, + * i.e. without any rotation applied, and provides zero-copied results via {@link ResultListener} + * in {@link FaceStylizerOptions}. Only use this method when the {@link FaceStylizer} is created + * with {@link RunningMode#IMAGE}. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + *

The image can be of any size. To ensure that the output image has reasonable quality, the + * size of the stylized output is based the model output size and can be smaller than the input + * image. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + * @throws MediaPipeException if there is an internal error. Or if {@link FaceStylizer} is not + * created wtih {@link ResultListener} set in {@link FaceStylizerOptions}. + */ + public void stylizeWithResultListener(MPImage image) { + stylizeWithResultListener(image, ImageProcessingOptions.builder().build()); + } + + /** + * Performs face stylization on the provided single image, and provides zero-copied results via + * {@link ResultListener} in {@link FaceStylizerOptions}. Only use this method when the {@link + * FaceStylizer} is created with {@link RunningMode#IMAGE}. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + *

The input image can be of any size, To ensure that the output image has reasonable quality, + * the stylized output image size is the smaller of the model output size and the size of the + * {@link ImageProcessingOptions#regionOfInterest} specified in {@code imageProcessingOptions}. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the + * input image before running inference. Note that region-of-interest is not supported + * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in + * this method throwing an IllegalArgumentException. + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + * @throws MediaPipeException if there is an internal error. Or if {@link FaceStylizer} is not + * created wtih {@link ResultListener} set in {@link FaceStylizerOptions}. + */ + public void stylizeWithResultListener( + MPImage image, ImageProcessingOptions imageProcessingOptions) { + if (!hasResultListener) { + throw new MediaPipeException( + MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), + "ResultListener is not set in the FaceStylizerOptions, but this method expects a" + + " ResultListener to process ImageSegmentationResult."); + } + var unused = processImageData(image, imageProcessingOptions); + } + + /** + * Performs face stylization on the provided video frame with default image processing options, + * i.e. without any rotation applied. Only use this method when the {@link FaceStylizer} is + * created with {@link RunningMode#VIDEO}. + * + *

It's required to provide the video frame's timestamp (in milliseconds). The input timestamps + * must be monotonically increasing. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + *

The image can be of any size. To ensure that the output image has reasonable quality, the + * size of the stylized output is based the model output size and can be smaller than the input + * image. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param timestampMs the input timestamp (in milliseconds). + * @throws MediaPipeException if there is an internal error. Or if {@link FaceStylizer} is created + * with a {@link ResultListener}. + */ + public FaceStylizerResult stylizeForVideo(MPImage image, long timestampMs) { + return stylizeForVideo(image, ImageProcessingOptions.builder().build(), timestampMs); + } + + /** + * Performs face stylization on the provided video frame. Only use this method when the {@link + * FaceStylizer} is created with {@link RunningMode#VIDEO}. + * + *

It's required to provide the video frame's timestamp (in milliseconds). The input timestamps + * must be monotonically increasing. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + *

The input image can be of any size, To ensure that the output image has reasonable quality, + * the stylized output image size is the smaller of the model output size and the size of the + * {@link ImageProcessingOptions#regionOfInterest} specified in {@code imageProcessingOptions}. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the + * input image before running inference. Note that region-of-interest is not supported + * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in + * this method throwing an IllegalArgumentException. + * @param timestampMs the input timestamp (in milliseconds). + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + * @throws MediaPipeException if there is an internal error. Or if {@link FaceStylizer} is created + * with a {@link ResultListener}. + */ + public FaceStylizerResult stylizeForVideo( + MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { + if (hasResultListener) { + throw new MediaPipeException( + MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), + "ResultListener is provided in the FaceStylizerOptions, but this method will return an" + + " ImageSegmentationResult."); + } + return (FaceStylizerResult) processVideoData(image, imageProcessingOptions, timestampMs); + } + + /** + * Performs face stylization on the provided video frame with default image processing options, + * i.e. without any rotation applied, and provides zero-copied results via {@link ResultListener} + * in {@link FaceStylizerOptions}. Only use this method when the {@link FaceStylizer} is created + * with {@link RunningMode#VIDEO}. + * + *

It's required to provide the video frame's timestamp (in milliseconds). The input timestamps + * must be monotonically increasing. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + *

The image can be of any size. To ensure that the output image has reasonable quality, the + * size of the stylized output is based the model output size and can be smaller than the input + * image. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param timestampMs the input timestamp (in milliseconds). + * @throws MediaPipeException if there is an internal error. Or if {@link FaceStylizer} is not + * created wtih {@link ResultListener} set in {@link FaceStylizerOptions}. + */ + public void stylizeForVideoWithResultListener(MPImage image, long timestampMs) { + stylizeForVideoWithResultListener(image, ImageProcessingOptions.builder().build(), timestampMs); + } + + /** + * Performs face stylization on the provided video frame, and provides zero-copied results via + * {@link ResultListener} in {@link FaceStylizerOptions}. Only use this method when the {@link + * FaceStylizer} is created with {@link RunningMode#VIDEO}. + * + *

It's required to provide the video frame's timestamp (in milliseconds). The input timestamps + * must be monotonically increasing. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + *

The input image can be of any size, To ensure that the output image has reasonable quality, + * the stylized output image size is the smaller of the model output size and the size of the + * {@link ImageProcessingOptions#regionOfInterest} specified in {@code imageProcessingOptions}. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param timestampMs the input timestamp (in milliseconds). + * @throws MediaPipeException if there is an internal error. Or if {@link FaceStylizer} is not + * created wtih {@link ResultListener} set in {@link FaceStylizerOptions}. + */ + public void stylizeForVideoWithResultListener( + MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { + if (!hasResultListener) { + throw new MediaPipeException( + MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), + "ResultListener is not set in the FaceStylizerOptions, but this method expects a" + + " ResultListener to process ImageSegmentationResult."); + } + var unused = processVideoData(image, imageProcessingOptions, timestampMs); + } + + /** + * Sends live image data to perform face stylization with default image processing options, i.e. + * without any rotation applied, and the results will be available via the {@link ResultListener} + * provided in the {@link FaceStylizerOptions}. Only use this method when the {@link FaceStylizer + * } is created with {@link RunningMode#LIVE_STREAM}. + * + *

It's required to provide a timestamp (in milliseconds) to indicate when the input image is + * sent to the face stylizer. The input timestamps must be monotonically increasing. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

The image can be of any size. To ensure that the output image has reasonable quality, the + * size of the stylized output is based the model output * size and can be smaller than the input + * image. + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param timestampMs the input timestamp (in milliseconds). + * @throws MediaPipeException if there is an internal error. + */ + public void stylizeAsync(MPImage image, long timestampMs) { + stylizeAsync(image, ImageProcessingOptions.builder().build(), timestampMs); + } + + /** + * Sends live image data to perform face stylization, and the results will be available via the + * {@link ResultListener} provided in the {@link FaceStylizerOptions}. Only use this method when + * the {@link FaceStylizer} is created with {@link RunningMode#LIVE_STREAM}. + * + *

It's required to provide a timestamp (in milliseconds) to indicate when the input image is + * sent to the face stylizer. The input timestamps must be monotonically increasing. + * + *

{@link FaceStylizer} supports the following color space types: + * + *

    + *
  • {@link android.graphics.Bitmap.Config#ARGB_8888} + *
+ * + *

The input image can be of any size, To ensure that the output image has reasonable quality, + * the stylized output image size is the smaller of the model output size and the size of the + * {@link ImageProcessingOptions#regionOfInterest} specified in {@code imageProcessingOptions}. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the + * input image before running inference. Note that region-of-interest is not supported + * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in + * this method throwing an IllegalArgumentException. + * @param timestampMs the input timestamp (in milliseconds). + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + * @throws MediaPipeException if there is an internal error. + */ + public void stylizeAsync( + MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { + sendLiveStreamData(image, imageProcessingOptions, timestampMs); + } + + /** Options for setting up an {@link FaceStylizer}. */ + @AutoValue + public abstract static class FaceStylizerOptions extends TaskOptions { + + /** Builder for {@link FaceStylizerOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Sets the base options for the face stylizer task. */ + public abstract Builder setBaseOptions(BaseOptions value); + + /** + * Sets the running mode for the face stylizer task. Default to the image mode. Image stylizer + * has three modes: + * + *

    + *
  • IMAGE: The mode for stylizeing image on single image inputs. + *
  • VIDEO: The mode for stylizeing image on the decoded frames of a video. + *
  • LIVE_STREAM: The mode for for stylizeing image on a live stream of input data, such + * as from camera. In this mode, {@code setResultListener} must be called to set up a + * listener to receive the recognition results asynchronously. + *
+ */ + public abstract Builder setRunningMode(RunningMode value); + + /** + * Sets an optional {@link ResultListener} to receive the stylization results when the graph + * pipeline is done processing an image. + */ + public abstract Builder setResultListener(ResultListener value); + + /** Sets an optional {@link ErrorListener}}. */ + public abstract Builder setErrorListener(ErrorListener value); + + abstract FaceStylizerOptions autoBuild(); + + /** + * Validates and builds the {@link FaceStylizerOptions} instance. + * + * @throws IllegalArgumentException if the result listener and the running mode are not + * properly configured. The result listener must be set when the face stylizer is in the + * live stream mode. + */ + public final FaceStylizerOptions build() { + FaceStylizerOptions options = autoBuild(); + if (options.runningMode() == RunningMode.LIVE_STREAM) { + if (!options.resultListener().isPresent()) { + throw new IllegalArgumentException( + "The face stylizer is in the live stream mode, a user-defined result listener" + + " must be provided in FaceStylizerOptions."); + } + } + return options; + } + } + + abstract BaseOptions baseOptions(); + + abstract RunningMode runningMode(); + + abstract Optional> resultListener(); + + abstract Optional errorListener(); + + public static Builder builder() { + return new AutoValue_FaceStylizer_FaceStylizerOptions.Builder() + .setRunningMode(RunningMode.IMAGE); + } + + /** Converts an {@link FaceStylizerOptions} to a {@link CalculatorOptions} protobuf message. */ + @Override + public CalculatorOptions convertToCalculatorOptionsProto() { + FaceStylizerGraphOptionsProto.FaceStylizerGraphOptions taskOptions = + FaceStylizerGraphOptionsProto.FaceStylizerGraphOptions.newBuilder() + .setBaseOptions( + BaseOptionsProto.BaseOptions.newBuilder() + .setUseStreamMode(runningMode() != RunningMode.IMAGE) + .mergeFrom(convertBaseOptionsToProto(baseOptions())) + .build()) + .build(); + + return CalculatorOptions.newBuilder() + .setExtension(FaceStylizerGraphOptionsProto.FaceStylizerGraphOptions.ext, taskOptions) + .build(); + } + } +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizerResult.java new file mode 100644 index 00000000..9235e473 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizerResult.java @@ -0,0 +1,39 @@ +// Copyright 2023 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.facestylizer; + +import com.google.auto.value.AutoValue; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.core.TaskResult; + +/** Represents the stylized image generated by {@link FaceStylizer}. */ +@AutoValue +public abstract class FaceStylizerResult implements TaskResult { + + /** + * Creates an {@link FaceStylizerResult} instance from a MPImage. + * + * @param stylizedImage an MPImage representing the stylized face. + * @param timestampMs a timestamp for this result. + */ + public static FaceStylizerResult create(MPImage stylizedImage, long timestampMs) { + return new AutoValue_FaceStylizerResult(stylizedImage, timestampMs); + } + + public abstract MPImage stylizedImage(); + + @Override + public abstract long timestampMs(); +} diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/AndroidManifest.xml b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/AndroidManifest.xml new file mode 100644 index 00000000..8fbc06e8 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/AndroidManifest.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/BUILD b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/BUILD new file mode 100644 index 00000000..c1448676 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/BUILD @@ -0,0 +1,19 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# TODO: Enable this in OSS diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizerTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizerTest.java new file mode 100644 index 00000000..98c7abb6 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/facestylizer/FaceStylizerTest.java @@ -0,0 +1,391 @@ +// Copyright 2023 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.facestylizer; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.content.res.AssetManager; +import android.graphics.BitmapFactory; +import android.graphics.RectF; +import android.util.Pair; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.mediapipe.framework.MediaPipeException; +import com.google.mediapipe.framework.image.BitmapImageBuilder; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.core.BaseOptions; +import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions; +import com.google.mediapipe.tasks.vision.core.RunningMode; +import com.google.mediapipe.tasks.vision.facestylizer.FaceStylizer.FaceStylizerOptions; +import java.io.InputStream; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; + +/** Test for {@link FaceStylizer}. */ +@RunWith(Suite.class) +@SuiteClasses({FaceStylizerTest.General.class, FaceStylizerTest.RunningModeTest.class}) +public class FaceStylizerTest { + private static final String modelFile = "face_stylization_dummy.tflite"; + private static final String testImage = "portrait.jpg"; + private static final int modelImageSize = 512; + + public Pair getRectPixelSize(MPImage originalImage, RectF rect) { + int width = originalImage.getWidth(); + int height = originalImage.getHeight(); + return new Pair<>( + (int) ((rect.right - rect.left) * width), (int) ((rect.bottom - rect.top) * height)); + } + + @RunWith(AndroidJUnit4.class) + public static final class General extends FaceStylizerTest { + FaceStylizer faceStylizer; + + @After + public void afterEach() throws Exception { + if (faceStylizer != null) { + faceStylizer.close(); + } + } + + @Test + public void create_succeeds() throws Exception { + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .build(); + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + assertThat(faceStylizer).isNotNull(); + } + + @Test + public void create_failsWithMissingModel() throws Exception { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().build()) + .build()); + assertThat(exception) + .hasMessageThat() + .contains( + "specify only one of the model asset path, the model asset file descriptor, and the" + + " model asset buffer"); + } + } + + @RunWith(AndroidJUnit4.class) + public static final class RunningModeTest extends FaceStylizerTest { + FaceStylizer faceStylizer; + + @After + public void afterEach() throws Exception { + if (faceStylizer != null) { + faceStylizer.close(); + } + } + + @Test + public void create_failsWithMissingResultListenerInLiveSteamMode() throws Exception { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.LIVE_STREAM) + .build()); + assertThat(exception) + .hasMessageThat() + .contains("a user-defined result listener must be provided"); + } + + @Test + public void stylizer_failsWithCallingWrongApiInImageMode() throws Exception { + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.IMAGE) + .build(); + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + MediaPipeException exception = + assertThrows( + MediaPipeException.class, + () -> + faceStylizer.stylizeForVideo( + getImageFromAsset(testImage), /* timestampsMs= */ 0)); + assertThat(exception).hasMessageThat().contains("not initialized with the video mode"); + exception = + assertThrows( + MediaPipeException.class, + () -> faceStylizer.stylizeAsync(getImageFromAsset(testImage), /* timestampsMs= */ 0)); + assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode"); + exception = + assertThrows( + MediaPipeException.class, + () -> faceStylizer.stylizeWithResultListener(getImageFromAsset(testImage))); + assertThat(exception) + .hasMessageThat() + .contains("ResultListener is not set in the FaceStylizerOptions"); + } + + @Test + public void stylizer_failsWithCallingWrongApiInVideoMode() throws Exception { + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.VIDEO) + .build(); + + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + MediaPipeException exception = + assertThrows( + MediaPipeException.class, () -> faceStylizer.stylize(getImageFromAsset(testImage))); + assertThat(exception).hasMessageThat().contains("not initialized with the image mode"); + exception = + assertThrows( + MediaPipeException.class, + () -> faceStylizer.stylizeAsync(getImageFromAsset(testImage), /* timestampsMs= */ 0)); + assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode"); + exception = + assertThrows( + MediaPipeException.class, + () -> + faceStylizer.stylizeForVideoWithResultListener( + getImageFromAsset(testImage), /* timestampsMs= */ 0)); + assertThat(exception) + .hasMessageThat() + .contains("ResultListener is not set in the FaceStylizerOptions"); + } + + @Test + public void stylizer_failsWithCallingWrongApiInLiveSteamMode() throws Exception { + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.LIVE_STREAM) + .setResultListener((result, inputImage) -> {}) + .build(); + + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + MediaPipeException exception = + assertThrows( + MediaPipeException.class, + () -> faceStylizer.stylizeWithResultListener(getImageFromAsset(testImage))); + assertThat(exception).hasMessageThat().contains("not initialized with the image mode"); + exception = + assertThrows( + MediaPipeException.class, + () -> + faceStylizer.stylizeForVideoWithResultListener( + getImageFromAsset(testImage), /* timestampsMs= */ 0)); + assertThat(exception).hasMessageThat().contains("not initialized with the video mode"); + } + + @Test + public void stylizer_succeedsWithImageMode() throws Exception { + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.IMAGE) + .build(); + + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + + MPImage inputImage = getImageFromAsset(testImage); + int inputWidth = inputImage.getWidth(); + int inputHeight = inputImage.getHeight(); + float inputAspectRatio = (float) inputWidth / inputHeight; + + FaceStylizerResult actualResult = faceStylizer.stylize(inputImage); + MPImage stylizedImage = actualResult.stylizedImage(); + assertThat(stylizedImage).isNotNull(); + assertThat(stylizedImage.getWidth()).isEqualTo((int) (modelImageSize * inputAspectRatio)); + assertThat(stylizedImage.getHeight()).isEqualTo(modelImageSize); + } + + @Test + public void stylizer_succeedsWithRegionOfInterest() throws Exception { + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.IMAGE) + .build(); + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + + MPImage inputImage = getImageFromAsset(testImage); + + // Region-of-interest around the face. + RectF roi = + new RectF(/* left= */ 0.32f, /* top= */ 0.02f, /* right= */ 0.67f, /* bottom= */ 0.32f); + ImageProcessingOptions imageProcessingOptions = + ImageProcessingOptions.builder().setRegionOfInterest(roi).build(); + + FaceStylizerResult actualResult = faceStylizer.stylize(inputImage, imageProcessingOptions); + var rectPixelSize = getRectPixelSize(inputImage, roi); + + MPImage stylizedImage = actualResult.stylizedImage(); + assertThat(stylizedImage).isNotNull(); + assertThat(stylizedImage.getWidth()).isEqualTo(rectPixelSize.first); + assertThat(stylizedImage.getHeight()).isEqualTo(rectPixelSize.second); + } + + @Test + public void stylizer_successWithImageModeWithResultListener() throws Exception { + MPImage inputImage = getImageFromAsset(testImage); + int inputWidth = inputImage.getWidth(); + int inputHeight = inputImage.getHeight(); + float inputAspectRatio = (float) inputWidth / inputHeight; + + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.IMAGE) + .setResultListener( + (result, originalImage) -> { + assertThat(originalImage).isEqualTo(inputImage); + + MPImage stylizedImage = result.stylizedImage(); + assertThat(stylizedImage).isNotNull(); + assertThat(stylizedImage.getWidth()) + .isEqualTo(modelImageSize * inputAspectRatio); + assertThat(stylizedImage.getHeight()).isEqualTo(modelImageSize); + }) + .build(); + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + faceStylizer.stylizeWithResultListener(getImageFromAsset(testImage)); + } + + @Test + public void stylizer_successWithVideoMode() throws Exception { + MPImage inputImage = getImageFromAsset(testImage); + int inputWidth = inputImage.getWidth(); + int inputHeight = inputImage.getHeight(); + float inputAspectRatio = (float) inputWidth / inputHeight; + + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.VIDEO) + .build(); + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + for (int i = 0; i < 3; i++) { + FaceStylizerResult actualResult = + faceStylizer.stylizeForVideo(getImageFromAsset(testImage), /* timestampsMs= */ i); + + MPImage stylizedImage = actualResult.stylizedImage(); + assertThat(stylizedImage).isNotNull(); + assertThat(stylizedImage.getWidth()).isEqualTo((int) (modelImageSize * inputAspectRatio)); + assertThat(stylizedImage.getHeight()).isEqualTo(modelImageSize); + } + } + + @Test + public void stylizer_successWithVideoModeWithResultListener() throws Exception { + MPImage inputImage = getImageFromAsset(testImage); + int inputWidth = inputImage.getWidth(); + int inputHeight = inputImage.getHeight(); + float inputAspectRatio = (float) inputWidth / inputHeight; + + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.VIDEO) + .setResultListener( + (result, originalImage) -> { + assertThat(originalImage).isEqualTo(inputImage); + + MPImage stylizedImage = result.stylizedImage(); + assertThat(stylizedImage).isNotNull(); + assertThat(stylizedImage.getWidth()) + .isEqualTo((int) (modelImageSize * inputAspectRatio)); + assertThat(stylizedImage.getHeight()).isEqualTo(modelImageSize); + }) + .build(); + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + for (int i = 0; i < 3; i++) { + faceStylizer.stylizeForVideoWithResultListener(inputImage, /* timestampsMs= */ i); + } + } + + @Test + public void stylizer_successWithLiveStreamMode() throws Exception { + MPImage inputImage = getImageFromAsset(testImage); + int inputWidth = inputImage.getWidth(); + int inputHeight = inputImage.getHeight(); + float inputAspectRatio = (float) inputWidth / inputHeight; + + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.LIVE_STREAM) + .setResultListener( + (result, originalImage) -> { + MPImage stylizedImage = result.stylizedImage(); + assertThat(stylizedImage).isNotNull(); + assertThat(stylizedImage.getWidth()) + .isEqualTo((int) (modelImageSize * inputAspectRatio)); + assertThat(stylizedImage.getHeight()).isEqualTo(modelImageSize); + }) + .build(); + + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + for (int i = 0; i < 3; i++) { + faceStylizer.stylizeAsync(inputImage, /* timestampsMs= */ i); + } + } + + @Test + public void stylizer_failsWithOutOfOrderInputTimestamps() throws Exception { + MPImage image = getImageFromAsset(testImage); + FaceStylizerOptions options = + FaceStylizerOptions.builder() + .setBaseOptions(BaseOptions.builder().setModelAssetPath(modelFile).build()) + .setRunningMode(RunningMode.LIVE_STREAM) + .setResultListener((result, inputImage) -> {}) + .build(); + faceStylizer = + FaceStylizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); + faceStylizer.stylizeAsync(image, /* timestampsMs= */ 1); + MediaPipeException exception = + assertThrows( + MediaPipeException.class, + () -> faceStylizer.stylizeAsync(image, /* timestampsMs= */ 0)); + assertThat(exception) + .hasMessageThat() + .contains("having a smaller timestamp than the processed timestamp"); + } + } + + private static MPImage getImageFromAsset(String filePath) throws Exception { + AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets(); + InputStream istr = assetManager.open(filePath); + return new BitmapImageBuilder(BitmapFactory.decodeStream(istr)).build(); + } +}