Project import generated by Copybara.

GitOrigin-RevId: 3864d8399e063cfcacc9e3f6e7ba653ea9045c52
2020-06-09 18:35:58 -04:00
parent b09cc090d4
commit 59ee17c1f3
19 changed files with 130 additions and 685 deletions
@@ -144,6 +144,10 @@ public class MainActivity extends AppCompatActivity {
    previewDisplayView.setVisibility(View.VISIBLE);
  }

+  protected Size cameraTargetResolution() {
+    return null; // No preference and let the camera (helper) decide.
+  }
+
  public void startCamera() {
    cameraHelper = new CameraXPreviewHelper();
    cameraHelper.setOnCameraStartedListener(
@@ -154,7 +158,30 @@ public class MainActivity extends AppCompatActivity {
        applicationInfo.metaData.getBoolean("cameraFacingFront", false)
            ? CameraHelper.CameraFacing.FRONT
            : CameraHelper.CameraFacing.BACK;
-    cameraHelper.startCamera(this, cameraFacing, /*surfaceTexture=*/ null);
+    cameraHelper.startCamera(
+        this, cameraFacing, /*surfaceTexture=*/ null, cameraTargetResolution());
+  }
+
+  protected Size computeViewSize(int width, int height) {
+    return new Size(width, height);
+  }
+
+  protected void onPreviewDisplaySurfaceChanged(
+      SurfaceHolder holder, int format, int width, int height) {
+    // (Re-)Compute the ideal size of the camera-preview display (the area that the
+    // camera-preview frames get rendered onto, potentially with scaling and rotation)
+    // based on the size of the SurfaceView that contains the display.
+    Size viewSize = computeViewSize(width, height);
+    Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+    boolean isCameraRotated = cameraHelper.isCameraRotated();
+
+    // Connect the converter to the camera-preview frames as its input (via
+    // previewFrameTexture), and configure the output width and height as the computed
+    // display size.
+    converter.setSurfaceTextureAndAttachToGLContext(
+        previewFrameTexture,
+        isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+        isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
  }

  private void setupPreviewDisplayView() {
@@ -173,20 +200,7 @@ public class MainActivity extends AppCompatActivity {

              @Override
              public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
-                // (Re-)Compute the ideal size of the camera-preview display (the area that the
-                // camera-preview frames get rendered onto, potentially with scaling and rotation)
-                // based on the size of the SurfaceView that contains the display.
-                Size viewSize = new Size(width, height);
-                Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
-                boolean isCameraRotated = cameraHelper.isCameraRotated();
-
-                // Connect the converter to the camera-preview frames as its input (via
-                // previewFrameTexture), and configure the output width and height as the computed
-                // display size.
-                converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture,
-                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
-                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
+                onPreviewDisplaySurfaceChanged(holder, format, width, height);
              }

              @Override
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    package="com.google.mediapipe.apps.objectdetection3d">
-
-  <uses-sdk
-      android:minSdkVersion="21"
-      android:targetSdkVersion="27" />
-
-  <!-- For using the camera -->
-  <uses-permission android:name="android.permission.CAMERA" />
-  <uses-feature android:name="android.hardware.camera" />
-  <uses-feature android:name="android.hardware.camera.autofocus" />
-  <!-- For MediaPipe -->
-  <uses-feature android:glEsVersion="0x00020000" android:required="true" />
-
-
-  <application
-      android:allowBackup="true"
-      android:label="@string/app_name"
-      android:supportsRtl="true"
-      android:theme="@style/AppTheme">
-      <activity
-          android:name=".MainActivity"
-          android:exported="true"
-          android:screenOrientation="portrait">
-          <intent-filter>
-              <action android:name="android.intent.action.MAIN" />
-              <category android:name="android.intent.category.LAUNCHER" />
-          </intent-filter>
-      </activity>
-  </application>
-
-</manifest>
@@ -41,16 +41,13 @@ config_setting(
    },
 )

-# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be
-# easily incorporated into the app via, for example,
-# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb".
 genrule(
    name = "binary_graph",
    srcs = select({
        "//conditions:default": ["//mediapipe/graphs/object_detection_3d:mobile_gpu_binary_graph_shoe"],
        ":use_chair_model": ["//mediapipe/graphs/object_detection_3d:mobile_gpu_binary_graph_chair"],
    }),
-    outs = ["objectdetection3d.binarypb"],
+    outs = ["object_detection_3d.binarypb"],
    cmd = "cp $< $@",
 )

@@ -64,8 +61,8 @@ genrule(
    cmd = "cp $< $@",
 )

-android_library(
-    name = "mediapipe_lib",
+android_binary(
+    name = "objectdetection3d",
    srcs = glob(["*.java"]),
    assets = [
        ":binary_graph",
@@ -83,33 +80,22 @@ android_library(
        ],
    }),
    assets_dir = "",
-    manifest = "AndroidManifest.xml",
-    resource_files = glob(["res/**"]),
-    deps = [
-        ":mediapipe_jni_lib",
-        "//mediapipe/framework/formats:landmark_java_proto_lite",
-        "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
-        "//mediapipe/java/com/google/mediapipe/components:android_components",
-        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
-        "//mediapipe/java/com/google/mediapipe/glutil",
-        "//third_party:androidx_appcompat",
-        "//third_party:androidx_constraint_layout",
-        "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_recyclerview",
-        "//third_party:opencv",
-        "@maven//:androidx_concurrent_concurrent_futures",
-        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@maven//:com_google_code_findbugs_jsr305",
-        "@maven//:com_google_guava_guava",
-    ],
-)
-
-android_binary(
-    name = "objectdetection3d",
-    manifest = "AndroidManifest.xml",
-    manifest_values = {"applicationId": "com.google.mediapipe.apps.objectdetection3d"},
+    manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",
+    manifest_values = {
+        "applicationId": "com.google.mediapipe.apps.objectdetection3d",
+        "appName": "Object Detection 3D",
+        "mainActivity": ".MainActivity",
+        "cameraFacingFront": "False",
+        "binaryGraphName": "object_detection_3d.binarypb",
+        "inputVideoStreamName": "input_video",
+        "outputVideoStreamName": "output_video",
+        "flipFramesVertically": "True",
+    },
    multidex = "native",
    deps = [
-        ":mediapipe_lib",
+        ":mediapipe_jni_lib",
+        "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib",
+        "//mediapipe/framework/formats:landmark_java_proto_lite",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
    ],
 )
@@ -16,102 +16,33 @@ package com.google.mediapipe.apps.objectdetection3d;

 import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
-import android.graphics.SurfaceTexture;
 import android.os.Bundle;
-import androidx.appcompat.app.AppCompatActivity;
 import android.util.Log;
 import android.util.Size;
 import android.view.SurfaceHolder;
-import android.view.SurfaceView;
-import android.view.View;
-import android.view.ViewGroup;
-import com.google.mediapipe.components.CameraHelper;
-import com.google.mediapipe.components.CameraXPreviewHelper;
-import com.google.mediapipe.components.ExternalTextureConverter;
-import com.google.mediapipe.components.FrameProcessor;
-import com.google.mediapipe.components.PermissionHelper;
 import com.google.mediapipe.framework.AndroidAssetUtil;
 import com.google.mediapipe.framework.AndroidPacketCreator;
 import com.google.mediapipe.framework.Packet;
-import com.google.mediapipe.glutil.EglManager;
 import java.io.InputStream;
 import java.util.HashMap;
 import java.util.Map;

-/** Main activity of MediaPipe example apps. */
-public class MainActivity extends AppCompatActivity {
+/** Main activity of MediaPipe object detection 3D app. */
+public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
  private static final String TAG = "MainActivity";

-  private static final String BINARY_GRAPH_NAME = "objectdetection3d.binarypb";
-  private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
-  private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
-
  private static final String OBJ_TEXTURE = "texture.bmp";
  private static final String OBJ_FILE = "model.obj.uuu";
  private static final String BOX_TEXTURE = "classic_colors.png";
  private static final String BOX_FILE = "box.obj.uuu";

-  private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
-
-  // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
-  // processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
-  // This is needed because OpenGL represents images assuming the image origin is at the bottom-left
-  // corner, whereas MediaPipe in general assumes the image origin is at top-left.
-  private static final boolean FLIP_FRAMES_VERTICALLY = true;
-
-  // Target resolution should be 4:3 for this application, as expected by the model and tracker.
-  private static final Size TARGET_RESOLUTION = new Size(1280, 960);
-
-  static {
-    // Load all native libraries needed by the app.
-    System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java3");
-  }
-
-  // {@link SurfaceTexture} where the camera-preview frames can be accessed.
-  private SurfaceTexture previewFrameTexture;
-  // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
-  private SurfaceView previewDisplayView;
-
-  // Creates and manages an {@link EGLContext}.
-  private EglManager eglManager;
-  // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
-  // frames onto a {@link Surface}.
-  private FrameProcessor processor;
-  // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
-  // consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
-  private ExternalTextureConverter converter;
-
-  // Handles camera access via the {@link CameraX} Jetpack support library.
-  private CameraXPreviewHelper cameraHelper;
-
  // Assets.
  private Bitmap objTexture = null;
  private Bitmap boxTexture = null;

-  Size cameraImageSize;
-
  @Override
  protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
-    setContentView(R.layout.activity_main);
-
-    previewDisplayView = new SurfaceView(this);
-    setupPreviewDisplayView();
-
-    // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
-    // binary graphs.
-    AndroidAssetUtil.initializeNativeAssetManager(this);
-
-    eglManager = new EglManager(null);
-    processor =
-        new FrameProcessor(
-            this,
-            eglManager.getNativeContext(),
-            BINARY_GRAPH_NAME,
-            INPUT_VIDEO_STREAM_NAME,
-            OUTPUT_VIDEO_STREAM_NAME);
-    processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);

    prepareDemoAssets();
    AndroidPacketCreator packetCreator = processor.getPacketCreator();
@@ -121,134 +52,60 @@ public class MainActivity extends AppCompatActivity {
    inputSidePackets.put("obj_texture", packetCreator.createRgbaImageFrame(objTexture));
    inputSidePackets.put("box_texture", packetCreator.createRgbaImageFrame(boxTexture));
    processor.setInputSidePackets(inputSidePackets);
-
-    PermissionHelper.checkAndRequestCameraPermissions(this);
  }

  @Override
-  protected void onResume() {
-    super.onResume();
-    converter = new ExternalTextureConverter(eglManager.getContext());
-    converter.setFlipY(FLIP_FRAMES_VERTICALLY);
-    converter.setConsumer(processor);
-    if (PermissionHelper.cameraPermissionsGranted(this)) {
-      startCamera();
-    }
+  protected Size cameraTargetResolution() {
+    return new Size(1280, 960); // Prefer 4:3 aspect ratio (camera size is in landscape).
  }

  @Override
-  protected void onPause() {
-    super.onPause();
-    converter.close();
+  protected Size computeViewSize(int width, int height) {
+    return new Size(height, height * 3 / 4); // Prefer 3:4 aspect ratio.
  }

  @Override
-  public void onRequestPermissionsResult(
-      int requestCode, String[] permissions, int[] grantResults) {
-    super.onRequestPermissionsResult(requestCode, permissions, grantResults);
-    PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
-  }
+  protected void onPreviewDisplaySurfaceChanged(
+      SurfaceHolder holder, int format, int width, int height) {
+    super.onPreviewDisplaySurfaceChanged(holder, format, width, height);

-  private void setupPreviewDisplayView() {
-    previewDisplayView.setVisibility(View.GONE);
-    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
-    viewGroup.addView(previewDisplayView);
+    boolean isCameraRotated = cameraHelper.isCameraRotated();
+    Size cameraImageSize = cameraHelper.getFrameSize();
+    processor.setOnWillAddFrameListener(
+        (timestamp) -> {
+          try {
+            int cameraTextureWidth =
+                isCameraRotated ? cameraImageSize.getHeight() : cameraImageSize.getWidth();
+            int cameraTextureHeight =
+                isCameraRotated ? cameraImageSize.getWidth() : cameraImageSize.getHeight();

-    previewDisplayView
-        .getHolder()
-        .addCallback(
-            new SurfaceHolder.Callback() {
-              @Override
-              public void surfaceCreated(SurfaceHolder holder) {
-                processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
-              }
+            // Find limiting side and scale to 3:4 aspect ratio
+            float aspectRatio = (float) cameraTextureWidth / (float) cameraTextureHeight;
+            if (aspectRatio > 3.0 / 4.0) {
+              // width too big
+              cameraTextureWidth = (int) ((float) cameraTextureHeight * 3.0 / 4.0);
+            } else {
+              // height too big
+              cameraTextureHeight = (int) ((float) cameraTextureWidth * 4.0 / 3.0);
+            }
+            Packet widthPacket = processor.getPacketCreator().createInt32(cameraTextureWidth);
+            Packet heightPacket = processor.getPacketCreator().createInt32(cameraTextureHeight);

-              @Override
-              public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
-                // (Re-)Compute the ideal size of the camera-preview display (the area that the
-                // camera-preview frames get rendered onto, potentially with scaling and rotation)
-                // based on the size of the SurfaceView that contains the display.
-                Size viewSize = new Size(height, height * 3 / 4); // Prefer 3:4 aspect ratio.
-                Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
-                boolean isCameraRotated = cameraHelper.isCameraRotated();
-                cameraImageSize = cameraHelper.getFrameSize();
-
-                // Connect the converter to the camera-preview frames as its input (via
-                // previewFrameTexture), and configure the output width and height as the computed
-                // display size.
-                converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture,
-                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
-                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
-                processor.setOnWillAddFrameListener(
-                    (timestamp) -> {
-                      try {
-                        int cameraTextureWidth =
-                            isCameraRotated
-                                ? cameraImageSize.getHeight()
-                                : cameraImageSize.getWidth();
-                        int cameraTextureHeight =
-                            isCameraRotated
-                                ? cameraImageSize.getWidth()
-                                : cameraImageSize.getHeight();
-
-                        // Find limiting side and scale to 3:4 aspect ratio
-                        float aspectRatio =
-                            (float) cameraTextureWidth / (float) cameraTextureHeight;
-                        if (aspectRatio > 3.0 / 4.0) {
-                          // width too big
-                          cameraTextureWidth = (int) ((float) cameraTextureHeight * 3.0 / 4.0);
-                        } else {
-                          // height too big
-                          cameraTextureHeight = (int) ((float) cameraTextureWidth * 4.0 / 3.0);
-                        }
-                        Packet widthPacket =
-                            processor.getPacketCreator().createInt32(cameraTextureWidth);
-                        Packet heightPacket =
-                            processor.getPacketCreator().createInt32(cameraTextureHeight);
-
-                        try {
-                          processor
-                              .getGraph()
-                              .addPacketToInputStream("input_width", widthPacket, timestamp);
-                          processor
-                              .getGraph()
-                              .addPacketToInputStream("input_height", heightPacket, timestamp);
-                        } catch (Exception e) {
-                          Log.e(
-                              TAG,
-                              "MediaPipeException encountered adding packets to width and height"
-                                  + " input streams.");
-                        }
-                        widthPacket.release();
-                        heightPacket.release();
-                      } catch (IllegalStateException ise) {
-                        Log.e(
-                            TAG,
-                            "Exception while adding packets to width and height input streams.");
-                      }
-                    });
-              }
-
-              @Override
-              public void surfaceDestroyed(SurfaceHolder holder) {
-                processor.getVideoSurfaceOutput().setSurface(null);
-              }
-            });
-  }
-
-  private void startCamera() {
-    cameraHelper = new CameraXPreviewHelper();
-    cameraHelper.setOnCameraStartedListener(
-        surfaceTexture -> {
-          previewFrameTexture = surfaceTexture;
-          // Make the display view visible to start showing the preview. This triggers the
-          // SurfaceHolder.Callback added to (the holder of) previewDisplayView.
-          previewDisplayView.setVisibility(View.VISIBLE);
+            try {
+              processor.getGraph().addPacketToInputStream("input_width", widthPacket, timestamp);
+              processor.getGraph().addPacketToInputStream("input_height", heightPacket, timestamp);
+            } catch (RuntimeException e) {
+              Log.e(
+                  TAG,
+                  "MediaPipeException encountered adding packets to width and height"
+                      + " input streams.");
+            }
+            widthPacket.release();
+            heightPacket.release();
+          } catch (IllegalStateException ise) {
+            Log.e(TAG, "Exception while adding packets to width and height input streams.");
+          }
        });
-    cameraHelper.startCamera(
-        this, CAMERA_FACING, /*surfaceTexture=*/ null, /*targetSize=*/ TARGET_RESOLUTION);
-    cameraImageSize = cameraHelper.getFrameSize();
  }

  private void prepareDemoAssets() {
@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
-    xmlns:app="http://schemas.android.com/apk/res-auto"
-    xmlns:tools="http://schemas.android.com/tools"
-    android:layout_width="match_parent"
-    android:layout_height="match_parent">
-
-    <FrameLayout
-        android:id="@+id/preview_display_layout"
-        android:layout_width="fill_parent"
-        android:layout_height="fill_parent"
-        android:layout_weight="1">
-        <TextView
-            android:id="@+id/no_camera_access_view"
-            android:layout_height="fill_parent"
-            android:layout_width="fill_parent"
-            android:gravity="center"
-            android:text="@string/no_camera_access" />
-    </FrameLayout>
-</androidx.constraintlayout.widget.ConstraintLayout>
@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<resources>
-    <color name="colorPrimary">#008577</color>
-    <color name="colorPrimaryDark">#00574B</color>
-    <color name="colorAccent">#D81B60</color>
-</resources>
@@ -1,4 +0,0 @@
-<resources>
-    <string name="app_name" translatable="false">Object Detection 3D</string>
-    <string name="no_camera_access" translatable="false">Please grant camera permissions.</string>
-</resources>
@@ -1,11 +0,0 @@
-<resources>
-
-    <!-- Base application theme. -->
-    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
-        <!-- Customize your theme here. -->
-        <item name="colorPrimary">@color/colorPrimary</item>
-        <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
-        <item name="colorAccent">@color/colorAccent</item>
-    </style>
-
-</resources>