Add support for image rotation in Java vision tasks.
PiperOrigin-RevId: 483493729
This commit is contained in:
MediaPipe Team
committed by
Copybara-Service
Copybara-Service
parent
b8502decff
commit
94cd134809
@@ -0,0 +1,24 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="com.google.mediapipe.tasks.vision.coretest"
|
||||
android:versionCode="1"
|
||||
android:versionName="1.0" >
|
||||
|
||||
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
|
||||
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
|
||||
|
||||
<uses-sdk android:minSdkVersion="24"
|
||||
android:targetSdkVersion="30" />
|
||||
|
||||
<application
|
||||
android:label="coretest"
|
||||
android:name="android.support.multidex.MultiDexApplication"
|
||||
android:taskAffinity="">
|
||||
<uses-library android:name="android.test.runner" />
|
||||
</application>
|
||||
|
||||
<instrumentation
|
||||
android:name="com.google.android.apps.common.testing.testrunner.GoogleInstrumentationTestRunner"
|
||||
android:targetPackage="com.google.mediapipe.tasks.vision.coretest" />
|
||||
|
||||
</manifest>
|
||||
@@ -0,0 +1,19 @@
|
||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
package(default_visibility = ["//mediapipe/tasks:internal"])
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
# TODO: Enable this in OSS
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.mediapipe.tasks.vision.core;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
|
||||
import android.graphics.RectF;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
/** Test for {@link ImageProcessingOptions}/ */
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public final class ImageProcessingOptionsTest {
|
||||
|
||||
@Test
|
||||
public void succeedsWithValidInputs() throws Exception {
|
||||
ImageProcessingOptions options =
|
||||
ImageProcessingOptions.builder()
|
||||
.setRegionOfInterest(new RectF(0.0f, 0.1f, 1.0f, 0.9f))
|
||||
.setRotationDegrees(270)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void failsWithLeftHigherThanRight() {
|
||||
IllegalArgumentException exception =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() ->
|
||||
ImageProcessingOptions.builder()
|
||||
.setRegionOfInterest(new RectF(0.9f, 0.0f, 0.1f, 1.0f))
|
||||
.build());
|
||||
assertThat(exception).hasMessageThat().contains("Expected left < right and top < bottom");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void failsWithBottomHigherThanTop() {
|
||||
IllegalArgumentException exception =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() ->
|
||||
ImageProcessingOptions.builder()
|
||||
.setRegionOfInterest(new RectF(0.0f, 0.9f, 1.0f, 0.1f))
|
||||
.build());
|
||||
assertThat(exception).hasMessageThat().contains("Expected left < right and top < bottom");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void failsWithInvalidRotation() {
|
||||
IllegalArgumentException exception =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> ImageProcessingOptions.builder().setRotationDegrees(1).build());
|
||||
assertThat(exception).hasMessageThat().contains("Expected rotation to be a multiple of 90°");
|
||||
}
|
||||
}
|
||||
+71
-8
@@ -19,6 +19,7 @@ import static org.junit.Assert.assertThrows;
|
||||
|
||||
import android.content.res.AssetManager;
|
||||
import android.graphics.BitmapFactory;
|
||||
import android.graphics.RectF;
|
||||
import androidx.test.core.app.ApplicationProvider;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.common.truth.Correspondence;
|
||||
@@ -30,6 +31,7 @@ import com.google.mediapipe.tasks.components.containers.Category;
|
||||
import com.google.mediapipe.tasks.components.containers.Landmark;
|
||||
import com.google.mediapipe.tasks.components.containers.proto.LandmarksDetectionResultProto.LandmarksDetectionResult;
|
||||
import com.google.mediapipe.tasks.core.BaseOptions;
|
||||
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
|
||||
import com.google.mediapipe.tasks.vision.core.RunningMode;
|
||||
import com.google.mediapipe.tasks.vision.gesturerecognizer.GestureRecognizer.GestureRecognizerOptions;
|
||||
import java.io.InputStream;
|
||||
@@ -46,11 +48,14 @@ public class GestureRecognizerTest {
|
||||
private static final String GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE = "gesture_recognizer.task";
|
||||
private static final String TWO_HANDS_IMAGE = "right_hands.jpg";
|
||||
private static final String THUMB_UP_IMAGE = "thumb_up.jpg";
|
||||
private static final String POINTING_UP_ROTATED_IMAGE = "pointing_up_rotated.jpg";
|
||||
private static final String NO_HANDS_IMAGE = "cats_and_dogs.jpg";
|
||||
private static final String THUMB_UP_LANDMARKS = "thumb_up_landmarks.pb";
|
||||
private static final String TAG = "Gesture Recognizer Test";
|
||||
private static final String THUMB_UP_LABEL = "Thumb_Up";
|
||||
private static final int THUMB_UP_INDEX = 5;
|
||||
private static final String POINTING_UP_LABEL = "Pointing_Up";
|
||||
private static final int POINTING_UP_INDEX = 3;
|
||||
private static final float LANDMARKS_ERROR_TOLERANCE = 0.03f;
|
||||
private static final int IMAGE_WIDTH = 382;
|
||||
private static final int IMAGE_HEIGHT = 406;
|
||||
@@ -135,6 +140,53 @@ public class GestureRecognizerTest {
|
||||
gestureRecognizer.recognize(getImageFromAsset(TWO_HANDS_IMAGE));
|
||||
assertThat(actualResult.handednesses()).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void recognize_successWithRotation() throws Exception {
|
||||
GestureRecognizerOptions options =
|
||||
GestureRecognizerOptions.builder()
|
||||
.setBaseOptions(
|
||||
BaseOptions.builder()
|
||||
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
|
||||
.build())
|
||||
.setNumHands(1)
|
||||
.build();
|
||||
GestureRecognizer gestureRecognizer =
|
||||
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
ImageProcessingOptions imageProcessingOptions =
|
||||
ImageProcessingOptions.builder().setRotationDegrees(-90).build();
|
||||
GestureRecognitionResult actualResult =
|
||||
gestureRecognizer.recognize(
|
||||
getImageFromAsset(POINTING_UP_ROTATED_IMAGE), imageProcessingOptions);
|
||||
assertThat(actualResult.gestures()).hasSize(1);
|
||||
assertThat(actualResult.gestures().get(0).get(0).index()).isEqualTo(POINTING_UP_INDEX);
|
||||
assertThat(actualResult.gestures().get(0).get(0).categoryName()).isEqualTo(POINTING_UP_LABEL);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void recognize_failsWithRegionOfInterest() throws Exception {
|
||||
GestureRecognizerOptions options =
|
||||
GestureRecognizerOptions.builder()
|
||||
.setBaseOptions(
|
||||
BaseOptions.builder()
|
||||
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
|
||||
.build())
|
||||
.setNumHands(1)
|
||||
.build();
|
||||
GestureRecognizer gestureRecognizer =
|
||||
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
ImageProcessingOptions imageProcessingOptions =
|
||||
ImageProcessingOptions.builder().setRegionOfInterest(new RectF(0, 0, 1, 1)).build();
|
||||
IllegalArgumentException exception =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() ->
|
||||
gestureRecognizer.recognize(
|
||||
getImageFromAsset(THUMB_UP_IMAGE), imageProcessingOptions));
|
||||
assertThat(exception)
|
||||
.hasMessageThat()
|
||||
.contains("GestureRecognizer doesn't support region-of-interest");
|
||||
}
|
||||
}
|
||||
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
@@ -195,12 +247,16 @@ public class GestureRecognizerTest {
|
||||
MediaPipeException exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> gestureRecognizer.recognizeForVideo(getImageFromAsset(THUMB_UP_IMAGE), 0));
|
||||
() ->
|
||||
gestureRecognizer.recognizeForVideo(
|
||||
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
|
||||
exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> gestureRecognizer.recognizeAsync(getImageFromAsset(THUMB_UP_IMAGE), 0));
|
||||
() ->
|
||||
gestureRecognizer.recognizeAsync(
|
||||
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
|
||||
}
|
||||
|
||||
@@ -225,7 +281,9 @@ public class GestureRecognizerTest {
|
||||
exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> gestureRecognizer.recognizeAsync(getImageFromAsset(THUMB_UP_IMAGE), 0));
|
||||
() ->
|
||||
gestureRecognizer.recognizeAsync(
|
||||
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
|
||||
}
|
||||
|
||||
@@ -251,7 +309,9 @@ public class GestureRecognizerTest {
|
||||
exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> gestureRecognizer.recognizeForVideo(getImageFromAsset(THUMB_UP_IMAGE), 0));
|
||||
() ->
|
||||
gestureRecognizer.recognizeForVideo(
|
||||
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
|
||||
}
|
||||
|
||||
@@ -291,7 +351,8 @@ public class GestureRecognizerTest {
|
||||
getExpectedGestureRecognitionResult(THUMB_UP_LANDMARKS, THUMB_UP_LABEL, THUMB_UP_INDEX);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
GestureRecognitionResult actualResult =
|
||||
gestureRecognizer.recognizeForVideo(getImageFromAsset(THUMB_UP_IMAGE), i);
|
||||
gestureRecognizer.recognizeForVideo(
|
||||
getImageFromAsset(THUMB_UP_IMAGE), /*timestampsMs=*/ i);
|
||||
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
|
||||
}
|
||||
}
|
||||
@@ -317,9 +378,11 @@ public class GestureRecognizerTest {
|
||||
.build();
|
||||
try (GestureRecognizer gestureRecognizer =
|
||||
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
|
||||
gestureRecognizer.recognizeAsync(image, 1);
|
||||
gestureRecognizer.recognizeAsync(image, /*timestampsMs=*/ 1);
|
||||
MediaPipeException exception =
|
||||
assertThrows(MediaPipeException.class, () -> gestureRecognizer.recognizeAsync(image, 0));
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> gestureRecognizer.recognizeAsync(image, /*timestampsMs=*/ 0));
|
||||
assertThat(exception)
|
||||
.hasMessageThat()
|
||||
.contains("having a smaller timestamp than the processed timestamp");
|
||||
@@ -348,7 +411,7 @@ public class GestureRecognizerTest {
|
||||
try (GestureRecognizer gestureRecognizer =
|
||||
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
|
||||
for (int i = 0; i < 3; i++) {
|
||||
gestureRecognizer.recognizeAsync(image, i);
|
||||
gestureRecognizer.recognizeAsync(image, /*timestampsMs=*/ i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+70
-9
@@ -29,6 +29,7 @@ import com.google.mediapipe.tasks.components.containers.Category;
|
||||
import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
|
||||
import com.google.mediapipe.tasks.core.BaseOptions;
|
||||
import com.google.mediapipe.tasks.core.TestUtils;
|
||||
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
|
||||
import com.google.mediapipe.tasks.vision.core.RunningMode;
|
||||
import com.google.mediapipe.tasks.vision.imageclassifier.ImageClassifier.ImageClassifierOptions;
|
||||
import java.io.InputStream;
|
||||
@@ -47,7 +48,9 @@ public class ImageClassifierTest {
|
||||
private static final String FLOAT_MODEL_FILE = "mobilenet_v2_1.0_224.tflite";
|
||||
private static final String QUANTIZED_MODEL_FILE = "mobilenet_v1_0.25_224_quant.tflite";
|
||||
private static final String BURGER_IMAGE = "burger.jpg";
|
||||
private static final String BURGER_ROTATED_IMAGE = "burger_rotated.jpg";
|
||||
private static final String MULTI_OBJECTS_IMAGE = "multi_objects.jpg";
|
||||
private static final String MULTI_OBJECTS_ROTATED_IMAGE = "multi_objects_rotated.jpg";
|
||||
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public static final class General extends ImageClassifierTest {
|
||||
@@ -209,13 +212,60 @@ public class ImageClassifierTest {
|
||||
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
// RectF around the soccer ball.
|
||||
RectF roi = new RectF(0.450f, 0.308f, 0.614f, 0.734f);
|
||||
ImageProcessingOptions imageProcessingOptions =
|
||||
ImageProcessingOptions.builder().setRegionOfInterest(roi).build();
|
||||
ImageClassificationResult results =
|
||||
imageClassifier.classify(getImageFromAsset(MULTI_OBJECTS_IMAGE), roi);
|
||||
imageClassifier.classify(getImageFromAsset(MULTI_OBJECTS_IMAGE), imageProcessingOptions);
|
||||
|
||||
assertHasOneHeadAndOneTimestamp(results, 0);
|
||||
assertCategoriesAre(
|
||||
results, Arrays.asList(Category.create(0.9969325f, 806, "soccer ball", "")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void classify_succeedsWithRotation() throws Exception {
|
||||
ImageClassifierOptions options =
|
||||
ImageClassifierOptions.builder()
|
||||
.setBaseOptions(BaseOptions.builder().setModelAssetPath(FLOAT_MODEL_FILE).build())
|
||||
.setClassifierOptions(ClassifierOptions.builder().setMaxResults(3).build())
|
||||
.build();
|
||||
ImageClassifier imageClassifier =
|
||||
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
ImageProcessingOptions imageProcessingOptions =
|
||||
ImageProcessingOptions.builder().setRotationDegrees(-90).build();
|
||||
ImageClassificationResult results =
|
||||
imageClassifier.classify(getImageFromAsset(BURGER_ROTATED_IMAGE), imageProcessingOptions);
|
||||
|
||||
assertHasOneHeadAndOneTimestamp(results, 0);
|
||||
assertCategoriesAre(
|
||||
results,
|
||||
Arrays.asList(
|
||||
Category.create(0.6390683f, 934, "cheeseburger", ""),
|
||||
Category.create(0.0495407f, 963, "meat loaf", ""),
|
||||
Category.create(0.0469720f, 925, "guacamole", "")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void classify_succeedsWithRegionOfInterestAndRotation() throws Exception {
|
||||
ImageClassifierOptions options =
|
||||
ImageClassifierOptions.builder()
|
||||
.setBaseOptions(BaseOptions.builder().setModelAssetPath(FLOAT_MODEL_FILE).build())
|
||||
.setClassifierOptions(ClassifierOptions.builder().setMaxResults(1).build())
|
||||
.build();
|
||||
ImageClassifier imageClassifier =
|
||||
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
// RectF around the chair.
|
||||
RectF roi = new RectF(0.0f, 0.1763f, 0.5642f, 0.3049f);
|
||||
ImageProcessingOptions imageProcessingOptions =
|
||||
ImageProcessingOptions.builder().setRegionOfInterest(roi).setRotationDegrees(-90).build();
|
||||
ImageClassificationResult results =
|
||||
imageClassifier.classify(
|
||||
getImageFromAsset(MULTI_OBJECTS_ROTATED_IMAGE), imageProcessingOptions);
|
||||
|
||||
assertHasOneHeadAndOneTimestamp(results, 0);
|
||||
assertCategoriesAre(
|
||||
results, Arrays.asList(Category.create(0.686824f, 560, "folding chair", "")));
|
||||
}
|
||||
}
|
||||
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
@@ -269,12 +319,16 @@ public class ImageClassifierTest {
|
||||
MediaPipeException exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> imageClassifier.classifyForVideo(getImageFromAsset(BURGER_IMAGE), 0));
|
||||
() ->
|
||||
imageClassifier.classifyForVideo(
|
||||
getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
|
||||
exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> imageClassifier.classifyAsync(getImageFromAsset(BURGER_IMAGE), 0));
|
||||
() ->
|
||||
imageClassifier.classifyAsync(
|
||||
getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
|
||||
}
|
||||
|
||||
@@ -296,7 +350,9 @@ public class ImageClassifierTest {
|
||||
exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> imageClassifier.classifyAsync(getImageFromAsset(BURGER_IMAGE), 0));
|
||||
() ->
|
||||
imageClassifier.classifyAsync(
|
||||
getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
|
||||
}
|
||||
|
||||
@@ -320,7 +376,9 @@ public class ImageClassifierTest {
|
||||
exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> imageClassifier.classifyForVideo(getImageFromAsset(BURGER_IMAGE), 0));
|
||||
() ->
|
||||
imageClassifier.classifyForVideo(
|
||||
getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
|
||||
}
|
||||
|
||||
@@ -352,7 +410,8 @@ public class ImageClassifierTest {
|
||||
ImageClassifier imageClassifier =
|
||||
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
ImageClassificationResult results = imageClassifier.classifyForVideo(image, i);
|
||||
ImageClassificationResult results =
|
||||
imageClassifier.classifyForVideo(image, /*timestampMs=*/ i);
|
||||
assertHasOneHeadAndOneTimestamp(results, i);
|
||||
assertCategoriesAre(
|
||||
results, Arrays.asList(Category.create(0.7952058f, 934, "cheeseburger", "")));
|
||||
@@ -377,9 +436,11 @@ public class ImageClassifierTest {
|
||||
.build();
|
||||
try (ImageClassifier imageClassifier =
|
||||
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
|
||||
imageClassifier.classifyAsync(getImageFromAsset(BURGER_IMAGE), 1);
|
||||
imageClassifier.classifyAsync(getImageFromAsset(BURGER_IMAGE), /*timestampMs=*/ 1);
|
||||
MediaPipeException exception =
|
||||
assertThrows(MediaPipeException.class, () -> imageClassifier.classifyAsync(image, 0));
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> imageClassifier.classifyAsync(image, /*timestampMs=*/ 0));
|
||||
assertThat(exception)
|
||||
.hasMessageThat()
|
||||
.contains("having a smaller timestamp than the processed timestamp");
|
||||
@@ -405,7 +466,7 @@ public class ImageClassifierTest {
|
||||
try (ImageClassifier imageClassifier =
|
||||
ImageClassifier.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
imageClassifier.classifyAsync(image, i);
|
||||
imageClassifier.classifyAsync(image, /*timestampMs=*/ i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+69
-16
@@ -29,6 +29,7 @@ import com.google.mediapipe.tasks.components.containers.Category;
|
||||
import com.google.mediapipe.tasks.components.containers.Detection;
|
||||
import com.google.mediapipe.tasks.core.BaseOptions;
|
||||
import com.google.mediapipe.tasks.core.TestUtils;
|
||||
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
|
||||
import com.google.mediapipe.tasks.vision.core.RunningMode;
|
||||
import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetector.ObjectDetectorOptions;
|
||||
import java.io.InputStream;
|
||||
@@ -45,10 +46,11 @@ import org.junit.runners.Suite.SuiteClasses;
|
||||
public class ObjectDetectorTest {
|
||||
private static final String MODEL_FILE = "coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.tflite";
|
||||
private static final String CAT_AND_DOG_IMAGE = "cats_and_dogs.jpg";
|
||||
private static final String CAT_AND_DOG_ROTATED_IMAGE = "cats_and_dogs_rotated.jpg";
|
||||
private static final int IMAGE_WIDTH = 1200;
|
||||
private static final int IMAGE_HEIGHT = 600;
|
||||
private static final float CAT_SCORE = 0.69f;
|
||||
private static final RectF catBoundingBox = new RectF(611, 164, 986, 596);
|
||||
private static final RectF CAT_BOUNDING_BOX = new RectF(611, 164, 986, 596);
|
||||
// TODO: Figure out why android_x86 and android_arm tests have slightly different
|
||||
// scores (0.6875 vs 0.69921875).
|
||||
private static final float SCORE_DIFF_TOLERANCE = 0.01f;
|
||||
@@ -67,7 +69,7 @@ public class ObjectDetectorTest {
|
||||
ObjectDetector objectDetector =
|
||||
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE));
|
||||
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
|
||||
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -104,7 +106,7 @@ public class ObjectDetectorTest {
|
||||
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE));
|
||||
// The score threshold should block all other other objects, except cat.
|
||||
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
|
||||
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -175,7 +177,7 @@ public class ObjectDetectorTest {
|
||||
ObjectDetector objectDetector =
|
||||
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE));
|
||||
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
|
||||
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -228,6 +230,46 @@ public class ObjectDetectorTest {
|
||||
.contains("`category_allowlist` and `category_denylist` are mutually exclusive options.");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void detect_succeedsWithRotation() throws Exception {
|
||||
ObjectDetectorOptions options =
|
||||
ObjectDetectorOptions.builder()
|
||||
.setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build())
|
||||
.setMaxResults(1)
|
||||
.setCategoryAllowlist(Arrays.asList("cat"))
|
||||
.build();
|
||||
ObjectDetector objectDetector =
|
||||
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
ImageProcessingOptions imageProcessingOptions =
|
||||
ImageProcessingOptions.builder().setRotationDegrees(-90).build();
|
||||
ObjectDetectionResult results =
|
||||
objectDetector.detect(
|
||||
getImageFromAsset(CAT_AND_DOG_ROTATED_IMAGE), imageProcessingOptions);
|
||||
|
||||
assertContainsOnlyCat(results, new RectF(22.0f, 611.0f, 452.0f, 890.0f), 0.7109375f);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void detect_failsWithRegionOfInterest() throws Exception {
|
||||
ObjectDetectorOptions options =
|
||||
ObjectDetectorOptions.builder()
|
||||
.setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build())
|
||||
.build();
|
||||
ObjectDetector objectDetector =
|
||||
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
ImageProcessingOptions imageProcessingOptions =
|
||||
ImageProcessingOptions.builder().setRegionOfInterest(new RectF(0, 0, 1, 1)).build();
|
||||
IllegalArgumentException exception =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() ->
|
||||
objectDetector.detect(
|
||||
getImageFromAsset(CAT_AND_DOG_IMAGE), imageProcessingOptions));
|
||||
assertThat(exception)
|
||||
.hasMessageThat()
|
||||
.contains("ObjectDetector doesn't support region-of-interest");
|
||||
}
|
||||
|
||||
// TODO: Implement detect_succeedsWithFloatImages, detect_succeedsWithOrientation,
|
||||
// detect_succeedsWithNumThreads, detect_successWithNumThreadsFromBaseOptions,
|
||||
// detect_failsWithInvalidNegativeNumThreads, detect_failsWithInvalidNumThreadsAsZero.
|
||||
@@ -282,12 +324,16 @@ public class ObjectDetectorTest {
|
||||
MediaPipeException exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> objectDetector.detectForVideo(getImageFromAsset(CAT_AND_DOG_IMAGE), 0));
|
||||
() ->
|
||||
objectDetector.detectForVideo(
|
||||
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
|
||||
exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> objectDetector.detectAsync(getImageFromAsset(CAT_AND_DOG_IMAGE), 0));
|
||||
() ->
|
||||
objectDetector.detectAsync(
|
||||
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
|
||||
}
|
||||
|
||||
@@ -309,7 +355,9 @@ public class ObjectDetectorTest {
|
||||
exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> objectDetector.detectAsync(getImageFromAsset(CAT_AND_DOG_IMAGE), 0));
|
||||
() ->
|
||||
objectDetector.detectAsync(
|
||||
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
|
||||
}
|
||||
|
||||
@@ -333,7 +381,9 @@ public class ObjectDetectorTest {
|
||||
exception =
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> objectDetector.detectForVideo(getImageFromAsset(CAT_AND_DOG_IMAGE), 0));
|
||||
() ->
|
||||
objectDetector.detectForVideo(
|
||||
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ 0));
|
||||
assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
|
||||
}
|
||||
|
||||
@@ -348,7 +398,7 @@ public class ObjectDetectorTest {
|
||||
ObjectDetector objectDetector =
|
||||
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE));
|
||||
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
|
||||
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -363,8 +413,9 @@ public class ObjectDetectorTest {
|
||||
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
ObjectDetectionResult results =
|
||||
objectDetector.detectForVideo(getImageFromAsset(CAT_AND_DOG_IMAGE), i);
|
||||
assertContainsOnlyCat(results, catBoundingBox, CAT_SCORE);
|
||||
objectDetector.detectForVideo(
|
||||
getImageFromAsset(CAT_AND_DOG_IMAGE), /*timestampsMs=*/ i);
|
||||
assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -377,16 +428,18 @@ public class ObjectDetectorTest {
|
||||
.setRunningMode(RunningMode.LIVE_STREAM)
|
||||
.setResultListener(
|
||||
(objectDetectionResult, inputImage) -> {
|
||||
assertContainsOnlyCat(objectDetectionResult, catBoundingBox, CAT_SCORE);
|
||||
assertContainsOnlyCat(objectDetectionResult, CAT_BOUNDING_BOX, CAT_SCORE);
|
||||
assertImageSizeIsExpected(inputImage);
|
||||
})
|
||||
.setMaxResults(1)
|
||||
.build();
|
||||
try (ObjectDetector objectDetector =
|
||||
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
|
||||
objectDetector.detectAsync(image, 1);
|
||||
objectDetector.detectAsync(image, /*timestampsMs=*/ 1);
|
||||
MediaPipeException exception =
|
||||
assertThrows(MediaPipeException.class, () -> objectDetector.detectAsync(image, 0));
|
||||
assertThrows(
|
||||
MediaPipeException.class,
|
||||
() -> objectDetector.detectAsync(image, /*timestampsMs=*/ 0));
|
||||
assertThat(exception)
|
||||
.hasMessageThat()
|
||||
.contains("having a smaller timestamp than the processed timestamp");
|
||||
@@ -402,7 +455,7 @@ public class ObjectDetectorTest {
|
||||
.setRunningMode(RunningMode.LIVE_STREAM)
|
||||
.setResultListener(
|
||||
(objectDetectionResult, inputImage) -> {
|
||||
assertContainsOnlyCat(objectDetectionResult, catBoundingBox, CAT_SCORE);
|
||||
assertContainsOnlyCat(objectDetectionResult, CAT_BOUNDING_BOX, CAT_SCORE);
|
||||
assertImageSizeIsExpected(inputImage);
|
||||
})
|
||||
.setMaxResults(1)
|
||||
@@ -410,7 +463,7 @@ public class ObjectDetectorTest {
|
||||
try (ObjectDetector objectDetector =
|
||||
ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options)) {
|
||||
for (int i = 0; i < 3; i++) {
|
||||
objectDetector.detectAsync(image, i);
|
||||
objectDetector.detectAsync(image, /*timestampsMs=*/ i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user