talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 373e3ac1e5839befd95bf7d73ceff3c5f1171969
This commit is contained in:
MediaPipe Team
2021-10-06 14:27:49 -07:00
committed by jqtang
parent 137e1cc763
commit 33d683c671
153 changed files with 7871 additions and 1349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/examples/android/solutions/create_win_symlinks.bat
+7
View File
@@ -12,5 +12,12 @@ cd /d %~dp0
cd facemesh\src\main
rm res
mklink /d res ..\..\..\res
@rem for face detection example app.
cd /d %~dp0
cd facedetection\src\main
rm res
mklink /d res ..\..\..\res
dir
pause
mediapipe/examples/android/solutions/facedetection/build.gradle
+51
View File
@@ -0,0 +1,51 @@
plugins {
id 'com.android.application'
}
android {
compileSdkVersion 30
buildToolsVersion "30.0.3"
defaultConfig {
applicationId "com.google.mediapipe.apps.facedetection"
minSdkVersion 21
targetSdkVersion 30
versionCode 1
versionName "1.0"
}
buildTypes {
release {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
}
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
}
dependencies {
implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar'])
implementation 'androidx.appcompat:appcompat:1.3.0'
implementation 'com.google.android.material:material:1.3.0'
implementation 'androidx.constraintlayout:constraintlayout:2.0.4'
implementation 'androidx.exifinterface:exifinterface:1.3.3'
testImplementation 'junit:junit:4.+'
androidTestImplementation 'androidx.test.ext:junit:1.1.2'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
// MediaPipe Face Detection Solution components.
implementation 'com.google.mediapipe:solution-core:latest.release'
implementation 'com.google.mediapipe:facedetection:latest.release'
// MediaPipe deps
implementation 'com.google.flogger:flogger:0.6'
implementation 'com.google.flogger:flogger-system-backend:0.6'
implementation 'com.google.guava:guava:27.0.1-android'
implementation 'com.google.protobuf:protobuf-java:3.11.4'
// CameraX core library
def camerax_version = "1.0.0-beta10"
implementation "androidx.camera:camera-core:$camerax_version"
implementation "androidx.camera:camera-camera2:$camerax_version"
implementation "androidx.camera:camera-lifecycle:$camerax_version"
}
mediapipe/examples/android/solutions/facedetection/proguard-rules.pro
Vendored
+21
View File
@@ -0,0 +1,21 @@
# Add project specific ProGuard rules here.
# You can control the set of applied configuration files using the
# proguardFiles setting in build.gradle.
#
# For more details, see
# http://developer.android.com/guide/developing/tools/proguard.html
# If your project uses WebView with JS, uncomment the following
# and specify the fully qualified class name to the JavaScript interface
# class:
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
# public *;
#}
# Uncomment this to preserve the line number information for
# debugging stack traces.
#-keepattributes SourceFile,LineNumberTable
# If you keep the line number information, uncomment this to
# hide the original source file name.
#-renamesourcefileattribute SourceFile
mediapipe/examples/android/solutions/facedetection/src/main/AndroidManifest.xml
+32
View File
@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.google.mediapipe.examples.facedetection">
<uses-sdk
android:minSdkVersion="21"
android:targetSdkVersion="30" />
<!-- For loading images from gallery -->
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
<!-- For using the camera -->
<uses-permission android:name="android.permission.CAMERA" />
<uses-feature android:name="android.hardware.camera" />
<application
android:allowBackup="true"
android:icon="@mipmap/ic_launcher"
android:label="MediaPipe Face Detection"
android:roundIcon="@mipmap/ic_launcher_round"
android:supportsRtl="true"
android:theme="@style/AppTheme">
<activity android:name=".MainActivity"
android:screenOrientation="portrait">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
</application>
</manifest>
mediapipe/examples/android/solutions/facedetection/src/main/BUILD
+46
View File
@@ -0,0 +1,46 @@
# Copyright 2021 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
android_binary(
name = "facedetection",
srcs = glob(["**/*.java"]),
custom_package = "com.google.mediapipe.examples.facedetection",
manifest = "AndroidManifest.xml",
manifest_values = {
"applicationId": "com.google.mediapipe.examples.facedetection",
},
multidex = "native",
resource_files = ["//mediapipe/examples/android/solutions:resource_files"],
deps = [
"//mediapipe/framework/formats:detection_java_proto_lite",
"//mediapipe/framework/formats:location_data_java_proto_lite",
"//mediapipe/java/com/google/mediapipe/solutioncore:camera_input",
"//mediapipe/java/com/google/mediapipe/solutioncore:mediapipe_jni_lib",
"//mediapipe/java/com/google/mediapipe/solutioncore:solution_rendering",
"//mediapipe/java/com/google/mediapipe/solutioncore:video_input",
"//mediapipe/java/com/google/mediapipe/solutions/facedetection",
"//third_party:androidx_appcompat",
"//third_party:androidx_constraint_layout",
"//third_party:opencv",
"@maven//:androidx_activity_activity",
"@maven//:androidx_concurrent_concurrent_futures",
"@maven//:androidx_exifinterface_exifinterface",
"@maven//:androidx_fragment_fragment",
"@maven//:com_google_guava_guava",
],
)
mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultGlRenderer.java
+146
View File
@@ -0,0 +1,146 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.examples.facedetection;
import android.opengl.GLES20;
import com.google.mediapipe.formats.proto.DetectionProto.Detection;
import com.google.mediapipe.solutioncore.ResultGlRenderer;
import com.google.mediapipe.solutions.facedetection.FaceDetectionResult;
import com.google.mediapipe.solutions.facedetection.FaceKeypoint;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
/** A custom implementation of {@link ResultGlRenderer} to render {@link FaceDetectionResult}. */
public class FaceDetectionResultGlRenderer implements ResultGlRenderer<FaceDetectionResult> {
private static final String TAG = "FaceDetectionResultGlRenderer";
private static final float[] KEYPOINT_COLOR = new float[] {1f, 0f, 0f, 1f};
private static final float KEYPOINT_SIZE = 16f;
private static final float[] BBOX_COLOR = new float[] {0f, 1f, 0f, 1f};
private static final int BBOX_THICKNESS = 8;
private static final String VERTEX_SHADER =
"uniform mat4 uProjectionMatrix;\n"
+ "uniform float uPointSize;\n"
+ "attribute vec4 vPosition;\n"
+ "void main() {\n"
+ " gl_Position = uProjectionMatrix * vPosition;\n"
+ " gl_PointSize = uPointSize;"
+ "}";
private static final String FRAGMENT_SHADER =
"precision mediump float;\n"
+ "uniform vec4 uColor;\n"
+ "void main() {\n"
+ " gl_FragColor = uColor;\n"
+ "}";
private int program;
private int positionHandle;
private int pointSizeHandle;
private int projectionMatrixHandle;
private int colorHandle;
private int loadShader(int type, String shaderCode) {
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
return shader;
}
@Override
public void setupRendering() {
program = GLES20.glCreateProgram();
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, VERTEX_SHADER);
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, FRAGMENT_SHADER);
GLES20.glAttachShader(program, vertexShader);
GLES20.glAttachShader(program, fragmentShader);
GLES20.glLinkProgram(program);
positionHandle = GLES20.glGetAttribLocation(program, "vPosition");
pointSizeHandle = GLES20.glGetUniformLocation(program, "uPointSize");
projectionMatrixHandle = GLES20.glGetUniformLocation(program, "uProjectionMatrix");
colorHandle = GLES20.glGetUniformLocation(program, "uColor");
}
@Override
public void renderResult(FaceDetectionResult result, float[] projectionMatrix) {
if (result == null) {
return;
}
GLES20.glUseProgram(program);
GLES20.glUniformMatrix4fv(projectionMatrixHandle, 1, false, projectionMatrix, 0);
GLES20.glUniform1f(pointSizeHandle, KEYPOINT_SIZE);
int numDetectedFaces = result.multiFaceDetections().size();
for (int i = 0; i < numDetectedFaces; ++i) {
drawDetection(result.multiFaceDetections().get(i));
}
}
/**
* Deletes the shader program.
*
* <p>This is only necessary if one wants to release the program while keeping the context around.
*/
public void release() {
GLES20.glDeleteProgram(program);
}
private void drawDetection(Detection detection) {
if (!detection.hasLocationData()) {
return;
}
// Draw keypoints.
float[] points = new float[FaceKeypoint.NUM_KEY_POINTS * 2];
for (int i = 0; i < FaceKeypoint.NUM_KEY_POINTS; ++i) {
points[2 * i] = detection.getLocationData().getRelativeKeypoints(i).getX();
points[2 * i + 1] = detection.getLocationData().getRelativeKeypoints(i).getY();
}
GLES20.glUniform4fv(colorHandle, 1, KEYPOINT_COLOR, 0);
FloatBuffer vertexBuffer =
ByteBuffer.allocateDirect(points.length * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
.put(points);
vertexBuffer.position(0);
GLES20.glEnableVertexAttribArray(positionHandle);
GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer);
GLES20.glDrawArrays(GLES20.GL_POINTS, 0, FaceKeypoint.NUM_KEY_POINTS);
if (!detection.getLocationData().hasRelativeBoundingBox()) {
return;
}
// Draw bounding box.
float left = detection.getLocationData().getRelativeBoundingBox().getXmin();
float top = detection.getLocationData().getRelativeBoundingBox().getYmin();
float right = left + detection.getLocationData().getRelativeBoundingBox().getWidth();
float bottom = top + detection.getLocationData().getRelativeBoundingBox().getHeight();
drawLine(top, left, top, right);
drawLine(bottom, left, bottom, right);
drawLine(top, left, bottom, left);
drawLine(top, right, bottom, right);
}
private void drawLine(float y1, float x1, float y2, float x2) {
GLES20.glUniform4fv(colorHandle, 1, BBOX_COLOR, 0);
GLES20.glLineWidth(BBOX_THICKNESS);
float[] vertex = {x1, y1, x2, y2};
FloatBuffer vertexBuffer =
ByteBuffer.allocateDirect(vertex.length * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
.put(vertex);
vertexBuffer.position(0);
GLES20.glEnableVertexAttribArray(positionHandle);
GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer);
GLES20.glDrawArrays(GLES20.GL_LINES, 0, 2);
}
}
mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultImageView.java
+108
View File
@@ -0,0 +1,108 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.examples.facedetection;
import static java.lang.Math.min;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Matrix;
import android.graphics.Paint;
import androidx.appcompat.widget.AppCompatImageView;
import com.google.mediapipe.formats.proto.DetectionProto.Detection;
import com.google.mediapipe.solutions.facedetection.FaceDetectionResult;
import com.google.mediapipe.solutions.facedetection.FaceKeypoint;
/** An ImageView implementation for displaying {@link FaceDetectionResult}. */
public class FaceDetectionResultImageView extends AppCompatImageView {
private static final String TAG = "FaceDetectionResultImageView";
private static final int KEYPOINT_COLOR = Color.RED;
private static final int KEYPOINT_RADIUS = 15;
private static final int BBOX_COLOR = Color.GREEN;
private static final int BBOX_THICKNESS = 10;
private Bitmap latest;
public FaceDetectionResultImageView(Context context) {
super(context);
setScaleType(AppCompatImageView.ScaleType.FIT_CENTER);
}
/**
* Sets a {@link FaceDetectionResult} to render.
*
* @param result a {@link FaceDetectionResult} object that contains the solution outputs and the
* input {@link Bitmap}.
*/
public void setFaceDetectionResult(FaceDetectionResult result) {
if (result == null) {
return;
}
Bitmap bmInput = result.inputBitmap();
int width = bmInput.getWidth();
int height = bmInput.getHeight();
latest = Bitmap.createBitmap(width, height, bmInput.getConfig());
Canvas canvas = new Canvas(latest);
canvas.drawBitmap(bmInput, new Matrix(), null);
int numDetectedFaces = result.multiFaceDetections().size();
for (int i = 0; i < numDetectedFaces; ++i) {
drawDetectionOnCanvas(result.multiFaceDetections().get(i), canvas, width, height);
}
}
/** Updates the image view with the latest {@link FaceDetectionResult}. */
public void update() {
postInvalidate();
if (latest != null) {
setImageBitmap(latest);
}
}
private void drawDetectionOnCanvas(Detection detection, Canvas canvas, int width, int height) {
if (!detection.hasLocationData()) {
return;
}
// Draw keypoints.
Paint keypointPaint = new Paint();
keypointPaint.setColor(KEYPOINT_COLOR);
for (int i = 0; i < FaceKeypoint.NUM_KEY_POINTS; ++i) {
int xPixel =
min(
(int) (detection.getLocationData().getRelativeKeypoints(i).getX() * width),
width - 1);
int yPixel =
min(
(int) (detection.getLocationData().getRelativeKeypoints(i).getY() * height),
height - 1);
canvas.drawCircle(xPixel, yPixel, KEYPOINT_RADIUS, keypointPaint);
}
if (!detection.getLocationData().hasRelativeBoundingBox()) {
return;
}
// Draw bounding box.
Paint bboxPaint = new Paint();
bboxPaint.setColor(BBOX_COLOR);
bboxPaint.setStyle(Paint.Style.STROKE);
bboxPaint.setStrokeWidth(BBOX_THICKNESS);
float left = detection.getLocationData().getRelativeBoundingBox().getXmin() * width;
float top = detection.getLocationData().getRelativeBoundingBox().getYmin() * height;
float right = left + detection.getLocationData().getRelativeBoundingBox().getWidth() * width;
float bottom = top + detection.getLocationData().getRelativeBoundingBox().getHeight() * height;
canvas.drawRect(left, top, right, bottom, bboxPaint);
}
}
mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/MainActivity.java
+341
View File
@@ -0,0 +1,341 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.examples.facedetection;
import android.content.Intent;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.os.Bundle;
import android.provider.MediaStore;
import androidx.appcompat.app.AppCompatActivity;
import android.util.Log;
import android.view.View;
import android.widget.Button;
import android.widget.FrameLayout;
import androidx.activity.result.ActivityResultLauncher;
import androidx.activity.result.contract.ActivityResultContracts;
import androidx.exifinterface.media.ExifInterface;
// ContentResolver dependency
import com.google.mediapipe.formats.proto.LocationDataProto.LocationData.RelativeKeypoint;
import com.google.mediapipe.solutioncore.CameraInput;
import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
import com.google.mediapipe.solutioncore.VideoInput;
import com.google.mediapipe.solutions.facedetection.FaceDetection;
import com.google.mediapipe.solutions.facedetection.FaceDetectionOptions;
import com.google.mediapipe.solutions.facedetection.FaceDetectionResult;
import com.google.mediapipe.solutions.facedetection.FaceKeypoint;
import java.io.IOException;
import java.io.InputStream;
/** Main activity of MediaPipe Face Detection app. */
public class MainActivity extends AppCompatActivity {
private static final String TAG = "MainActivity";
private FaceDetection faceDetection;
private enum InputSource {
UNKNOWN,
IMAGE,
VIDEO,
CAMERA,
}
private InputSource inputSource = InputSource.UNKNOWN;
// Image demo UI and image loader components.
private ActivityResultLauncher<Intent> imageGetter;
private FaceDetectionResultImageView imageView;
// Video demo UI and video loader components.
private VideoInput videoInput;
private ActivityResultLauncher<Intent> videoGetter;
// Live camera demo UI and camera components.
private CameraInput cameraInput;
private SolutionGlSurfaceView<FaceDetectionResult> glSurfaceView;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
setupStaticImageDemoUiComponents();
setupVideoDemoUiComponents();
setupLiveDemoUiComponents();
}
@Override
protected void onResume() {
super.onResume();
if (inputSource == InputSource.CAMERA) {
// Restarts the camera and the opengl surface rendering.
cameraInput = new CameraInput(this);
cameraInput.setNewFrameListener(textureFrame -> faceDetection.send(textureFrame));
glSurfaceView.post(this::startCamera);
glSurfaceView.setVisibility(View.VISIBLE);
} else if (inputSource == InputSource.VIDEO) {
videoInput.resume();
}
}
@Override
protected void onPause() {
super.onPause();
if (inputSource == InputSource.CAMERA) {
glSurfaceView.setVisibility(View.GONE);
cameraInput.close();
} else if (inputSource == InputSource.VIDEO) {
videoInput.pause();
}
}
/** Sets up the UI components for the static image demo. */
private void setupStaticImageDemoUiComponents() {
// The Intent to access gallery and read images as bitmap.
imageGetter =
registerForActivityResult(
new ActivityResultContracts.StartActivityForResult(),
result -> {
Intent resultIntent = result.getData();
if (resultIntent != null) {
if (result.getResultCode() == RESULT_OK) {
Bitmap bitmap = null;
try {
bitmap =
MediaStore.Images.Media.getBitmap(
this.getContentResolver(), resultIntent.getData());
} catch (IOException e) {
Log.e(TAG, "Bitmap reading error:" + e);
}
try {
InputStream imageData =
this.getContentResolver().openInputStream(resultIntent.getData());
int orientation =
new ExifInterface(imageData)
.getAttributeInt(
ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
if (orientation != ExifInterface.ORIENTATION_NORMAL) {
Matrix matrix = new Matrix();
switch (orientation) {
case ExifInterface.ORIENTATION_ROTATE_90:
matrix.postRotate(90);
break;
case ExifInterface.ORIENTATION_ROTATE_180:
matrix.postRotate(180);
break;
case ExifInterface.ORIENTATION_ROTATE_270:
matrix.postRotate(270);
break;
default:
matrix.postRotate(0);
}
bitmap =
Bitmap.createBitmap(
bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
}
} catch (IOException e) {
Log.e(TAG, "Bitmap rotation error:" + e);
}
if (bitmap != null) {
faceDetection.send(bitmap);
}
}
}
});
Button loadImageButton = findViewById(R.id.button_load_picture);
loadImageButton.setOnClickListener(
v -> {
if (inputSource != InputSource.IMAGE) {
stopCurrentPipeline();
setupStaticImageModePipeline();
}
// Reads images from gallery.
Intent gallery =
new Intent(Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
imageGetter.launch(gallery);
});
imageView = new FaceDetectionResultImageView(this);
}
/** Sets up core workflow for static image mode. */
private void setupStaticImageModePipeline() {
this.inputSource = InputSource.IMAGE;
// Initializes a new MediaPipe Face Detection solution instance in the static image mode.
faceDetection =
new FaceDetection(
this,
FaceDetectionOptions.builder()
.setStaticImageMode(true)
.setModelSelection(0)
.setMinDetectionConfidence(0.5f)
.build());
// Connects MediaPipe Face Detection solution to the user-defined FaceDetectionResultImageView.
faceDetection.setResultListener(
faceDetectionResult -> {
logNoseTipKeypoint(faceDetectionResult, /*faceIndex=*/ 0, /*showPixelValues=*/ true);
imageView.setFaceDetectionResult(faceDetectionResult);
runOnUiThread(() -> imageView.update());
});
faceDetection.setErrorListener(
(message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
// Updates the preview layout.
FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
frameLayout.removeAllViewsInLayout();
imageView.setImageDrawable(null);
frameLayout.addView(imageView);
imageView.setVisibility(View.VISIBLE);
}
/** Sets up the UI components for the video demo. */
private void setupVideoDemoUiComponents() {
// The Intent to access gallery and read a video file.
videoGetter =
registerForActivityResult(
new ActivityResultContracts.StartActivityForResult(),
result -> {
Intent resultIntent = result.getData();
if (resultIntent != null) {
if (result.getResultCode() == RESULT_OK) {
glSurfaceView.post(
() ->
videoInput.start(
this,
resultIntent.getData(),
faceDetection.getGlContext(),
glSurfaceView.getWidth(),
glSurfaceView.getHeight()));
}
}
});
Button loadVideoButton = findViewById(R.id.button_load_video);
loadVideoButton.setOnClickListener(
v -> {
stopCurrentPipeline();
setupStreamingModePipeline(InputSource.VIDEO);
// Reads video from gallery.
Intent gallery =
new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
videoGetter.launch(gallery);
});
}
/** Sets up the UI components for the live demo with camera input. */
private void setupLiveDemoUiComponents() {
Button startCameraButton = findViewById(R.id.button_start_camera);
startCameraButton.setOnClickListener(
v -> {
if (inputSource == InputSource.CAMERA) {
return;
}
stopCurrentPipeline();
setupStreamingModePipeline(InputSource.CAMERA);
});
}
/** Sets up core workflow for streaming mode. */
private void setupStreamingModePipeline(InputSource inputSource) {
this.inputSource = inputSource;
// Initializes a new MediaPipe Face Detection solution instance in the streaming mode.
faceDetection =
new FaceDetection(
this,
FaceDetectionOptions.builder().setStaticImageMode(false).setModelSelection(0).build());
faceDetection.setErrorListener(
(message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
if (inputSource == InputSource.CAMERA) {
cameraInput = new CameraInput(this);
cameraInput.setNewFrameListener(textureFrame -> faceDetection.send(textureFrame));
} else if (inputSource == InputSource.VIDEO) {
videoInput = new VideoInput(this);
videoInput.setNewFrameListener(textureFrame -> faceDetection.send(textureFrame));
}
// Initializes a new Gl surface view with a user-defined FaceDetectionResultGlRenderer.
glSurfaceView =
new SolutionGlSurfaceView<>(
this, faceDetection.getGlContext(), faceDetection.getGlMajorVersion());
glSurfaceView.setSolutionResultRenderer(new FaceDetectionResultGlRenderer());
glSurfaceView.setRenderInputImage(true);
faceDetection.setResultListener(
faceDetectionResult -> {
logNoseTipKeypoint(faceDetectionResult, /*faceIndex=*/ 0, /*showPixelValues=*/ false);
glSurfaceView.setRenderData(faceDetectionResult);
glSurfaceView.requestRender();
});
// The runnable to start camera after the gl surface view is attached.
// For video input source, videoInput.start() will be called when the video uri is available.
if (inputSource == InputSource.CAMERA) {
glSurfaceView.post(this::startCamera);
}
// Updates the preview layout.
FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
imageView.setVisibility(View.GONE);
frameLayout.removeAllViewsInLayout();
frameLayout.addView(glSurfaceView);
glSurfaceView.setVisibility(View.VISIBLE);
frameLayout.requestLayout();
}
private void startCamera() {
cameraInput.start(
this,
faceDetection.getGlContext(),
CameraInput.CameraFacing.FRONT,
glSurfaceView.getWidth(),
glSurfaceView.getHeight());
}
private void stopCurrentPipeline() {
if (cameraInput != null) {
cameraInput.setNewFrameListener(null);
cameraInput.close();
}
if (videoInput != null) {
videoInput.setNewFrameListener(null);
videoInput.close();
}
if (glSurfaceView != null) {
glSurfaceView.setVisibility(View.GONE);
}
if (faceDetection != null) {
faceDetection.close();
}
}
private void logNoseTipKeypoint(
FaceDetectionResult result, int faceIndex, boolean showPixelValues) {
RelativeKeypoint noseTip =
FaceDetection.getFaceKeypoint(result, faceIndex, FaceKeypoint.NOSE_TIP);
// For Bitmaps, show the pixel values. For texture inputs, show the normalized coordinates.
if (showPixelValues) {
int width = result.inputBitmap().getWidth();
int height = result.inputBitmap().getHeight();
Log.i(
TAG,
String.format(
"MediaPipe Face Detection nose tip coordinates (pixel values): x=%f, y=%f",
noseTip.getX() * width, noseTip.getY() * height));
} else {
Log.i(
TAG,
String.format(
"MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]):"
+ " x=%f, y=%f",
noseTip.getX(), noseTip.getY()));
}
}
}
mediapipe/examples/android/solutions/facedetection/src/main/res
Symlink
+1
View File
@@ -0,0 +1 @@
../../../res
mediapipe/examples/android/solutions/facemesh/build.gradle
+4 -3
View File
@@ -31,15 +31,16 @@ dependencies {
implementation 'androidx.appcompat:appcompat:1.3.0'
implementation 'com.google.android.material:material:1.3.0'
implementation 'androidx.constraintlayout:constraintlayout:2.0.4'
implementation 'androidx.exifinterface:exifinterface:1.3.3'
testImplementation 'junit:junit:4.+'
androidTestImplementation 'androidx.test.ext:junit:1.1.2'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
// MediaPipe hands solution API and solution-core.
// MediaPipe Face Mesh Solution components.
implementation 'com.google.mediapipe:solution-core:latest.release'
implementation 'com.google.mediapipe:facemesh:latest.release'
// MediaPipe deps
implementation 'com.google.flogger:flogger:latest.release'
implementation 'com.google.flogger:flogger-system-backend:latest.release'
implementation 'com.google.flogger:flogger:0.6'
implementation 'com.google.flogger:flogger-system-backend:0.6'
implementation 'com.google.guava:guava:27.0.1-android'
implementation 'com.google.protobuf:protobuf-java:3.11.4'
// CameraX core library
mediapipe/examples/android/solutions/facemesh/src/main/BUILD
+1
View File
@@ -38,6 +38,7 @@ android_binary(
"//third_party:opencv",
"@maven//:androidx_activity_activity",
"@maven//:androidx_concurrent_concurrent_futures",
"@maven//:androidx_exifinterface_exifinterface",
"@maven//:androidx_fragment_fragment",
"@maven//:com_google_guava_guava",
],
mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultGlRenderer.java
+24 -34
View File
@@ -15,11 +15,10 @@
package com.google.mediapipe.examples.facemesh;
import android.opengl.GLES20;
import android.opengl.Matrix;
import com.google.common.collect.ImmutableSet;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
import com.google.mediapipe.solutioncore.ResultGlBoundary;
import com.google.mediapipe.solutioncore.ResultGlRenderer;
import com.google.mediapipe.solutions.facemesh.FaceMesh;
import com.google.mediapipe.solutions.facemesh.FaceMeshConnections;
import com.google.mediapipe.solutions.facemesh.FaceMeshResult;
import java.nio.ByteBuffer;
@@ -27,7 +26,7 @@ import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.util.List;
/** A custom implementation of {@link ResultGlRenderer} to render MediaPope FaceMesh results. */
/** A custom implementation of {@link ResultGlRenderer} to render {@link FaceMeshResult}. */
public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult> {
private static final String TAG = "FaceMeshResultGlRenderer";
@@ -46,10 +45,10 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
private static final float[] LIPS_COLOR = new float[] {0.9f, 0.9f, 0.9f, 1f};
private static final int LIPS_THICKNESS = 8;
private static final String VERTEX_SHADER =
"uniform mat4 uTransformMatrix;\n"
"uniform mat4 uProjectionMatrix;\n"
+ "attribute vec4 vPosition;\n"
+ "void main() {\n"
+ " gl_Position = uTransformMatrix * vPosition;\n"
+ " gl_Position = uProjectionMatrix * vPosition;\n"
+ "}";
private static final String FRAGMENT_SHADER =
"precision mediump float;\n"
@@ -59,9 +58,8 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
+ "}";
private int program;
private int positionHandle;
private int transformMatrixHandle;
private int projectionMatrixHandle;
private int colorHandle;
private final float[] transformMatrix = new float[16];
private int loadShader(int type, String shaderCode) {
int shader = GLES20.glCreateShader(type);
@@ -79,28 +77,17 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
GLES20.glAttachShader(program, fragmentShader);
GLES20.glLinkProgram(program);
positionHandle = GLES20.glGetAttribLocation(program, "vPosition");
transformMatrixHandle = GLES20.glGetUniformLocation(program, "uTransformMatrix");
projectionMatrixHandle = GLES20.glGetUniformLocation(program, "uProjectionMatrix");
colorHandle = GLES20.glGetUniformLocation(program, "uColor");
}
@Override
public void renderResult(FaceMeshResult result, ResultGlBoundary boundary) {
public void renderResult(FaceMeshResult result, float[] projectionMatrix) {
if (result == null) {
return;
}
GLES20.glUseProgram(program);
// Sets the transform matrix to align the result rendering with the scaled output texture.
// Also flips the rendering vertically since OpenGL assumes the coordinate origin is at the
// bottom-left corner, whereas MediaPipe landmark data assumes the coordinate origin is at the
// top-left corner.
Matrix.setIdentityM(transformMatrix, 0);
Matrix.scaleM(
transformMatrix,
0,
2 / (boundary.right() - boundary.left()),
-2 / (boundary.top() - boundary.bottom()),
1.0f);
GLES20.glUniformMatrix4fv(transformMatrixHandle, 1, false, transformMatrix, 0);
GLES20.glUniformMatrix4fv(projectionMatrixHandle, 1, false, projectionMatrix, 0);
int numFaces = result.multiFaceLandmarks().size();
for (int i = 0; i < numFaces; ++i) {
@@ -126,7 +113,7 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
LEFT_EYE_THICKNESS);
drawLandmarks(
result.multiFaceLandmarks().get(i).getLandmarkList(),
FaceMeshConnections.FACEMESH_LEFT_EYEBR0W,
FaceMeshConnections.FACEMESH_LEFT_EYEBROW,
LEFT_EYEBROW_COLOR,
LEFT_EYEBROW_THICKNESS);
drawLandmarks(
@@ -139,11 +126,24 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
FaceMeshConnections.FACEMESH_LIPS,
LIPS_COLOR,
LIPS_THICKNESS);
if (result.multiFaceLandmarks().get(i).getLandmarkCount()
== FaceMesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES) {
drawLandmarks(
result.multiFaceLandmarks().get(i).getLandmarkList(),
FaceMeshConnections.FACEMESH_RIGHT_IRIS,
RIGHT_EYE_COLOR,
RIGHT_EYE_THICKNESS);
drawLandmarks(
result.multiFaceLandmarks().get(i).getLandmarkList(),
FaceMeshConnections.FACEMESH_LEFT_IRIS,
LEFT_EYE_COLOR,
LEFT_EYE_THICKNESS);
}
}
}
/**
* Calls this to delete the shader program.
* Deletes the shader program.
*
* <p>This is only necessary if one wants to release the program while keeping the context around.
*/
@@ -159,13 +159,9 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
GLES20.glUniform4fv(colorHandle, 1, colorArray, 0);
GLES20.glLineWidth(thickness);
for (FaceMeshConnections.Connection c : connections) {
float[] vertex = new float[4];
NormalizedLandmark start = faceLandmarkList.get(c.start());
vertex[0] = normalizedLandmarkValue(start.getX());
vertex[1] = normalizedLandmarkValue(start.getY());
NormalizedLandmark end = faceLandmarkList.get(c.end());
vertex[2] = normalizedLandmarkValue(end.getX());
vertex[3] = normalizedLandmarkValue(end.getY());
float[] vertex = {start.getX(), start.getY(), end.getX(), end.getY()};
FloatBuffer vertexBuffer =
ByteBuffer.allocateDirect(vertex.length * 4)
.order(ByteOrder.nativeOrder())
@@ -177,10 +173,4 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
GLES20.glDrawArrays(GLES20.GL_LINES, 0, 2);
}
}
// Normalizes the value from the landmark value range:[0, 1] to the standard OpenGL coordinate
// value range: [-1, 1].
private float normalizedLandmarkValue(float value) {
return value * 2 - 1;
}
}
mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultImageView.java
+21 -4
View File
@@ -24,11 +24,12 @@ import androidx.appcompat.widget.AppCompatImageView;
import android.util.Size;
import com.google.common.collect.ImmutableSet;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
import com.google.mediapipe.solutions.facemesh.FaceMesh;
import com.google.mediapipe.solutions.facemesh.FaceMeshConnections;
import com.google.mediapipe.solutions.facemesh.FaceMeshResult;
import java.util.List;
/** An ImageView implementation for displaying MediaPipe FaceMesh results. */
/** An ImageView implementation for displaying {@link FaceMeshResult}. */
public class FaceMeshResultImageView extends AppCompatImageView {
private static final String TAG = "FaceMeshResultImageView";
@@ -103,7 +104,7 @@ public class FaceMeshResultImageView extends AppCompatImageView {
drawLandmarksOnCanvas(
canvas,
result.multiFaceLandmarks().get(i).getLandmarkList(),
FaceMeshConnections.FACEMESH_LEFT_EYEBR0W,
FaceMeshConnections.FACEMESH_LEFT_EYEBROW,
imageSize,
LEFT_EYEBROW_COLOR,
LEFT_EYEBROW_THICKNESS);
@@ -121,10 +122,27 @@ public class FaceMeshResultImageView extends AppCompatImageView {
imageSize,
LIPS_COLOR,
LIPS_THICKNESS);
if (result.multiFaceLandmarks().get(i).getLandmarkCount()
== FaceMesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES) {
drawLandmarksOnCanvas(
canvas,
result.multiFaceLandmarks().get(i).getLandmarkList(),
FaceMeshConnections.FACEMESH_RIGHT_IRIS,
imageSize,
RIGHT_EYE_COLOR,
RIGHT_EYE_THICKNESS);
drawLandmarksOnCanvas(
canvas,
result.multiFaceLandmarks().get(i).getLandmarkList(),
FaceMeshConnections.FACEMESH_LEFT_IRIS,
imageSize,
LEFT_EYE_COLOR,
LEFT_EYE_THICKNESS);
}
}
}
/** Updates the image view with the latest facemesh result. */
/** Updates the image view with the latest {@link FaceMeshResult}. */
public void update() {
postInvalidate();
if (latest != null) {
@@ -132,7 +150,6 @@ public class FaceMeshResultImageView extends AppCompatImageView {
}
}
// TODO: Better hand landmark and hand connection drawing.
private void drawLandmarksOnCanvas(
Canvas canvas,
List<NormalizedLandmark> faceLandmarkList,
mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/MainActivity.java
+49 -14
View File
@@ -16,6 +16,7 @@ package com.google.mediapipe.examples.facemesh;
import android.content.Intent;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.os.Bundle;
import android.provider.MediaStore;
import androidx.appcompat.app.AppCompatActivity;
@@ -25,6 +26,8 @@ import android.widget.Button;
import android.widget.FrameLayout;
import androidx.activity.result.ActivityResultLauncher;
import androidx.activity.result.contract.ActivityResultContracts;
import androidx.exifinterface.media.ExifInterface;
// ContentResolver dependency
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
import com.google.mediapipe.solutioncore.CameraInput;
import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
@@ -33,8 +36,9 @@ import com.google.mediapipe.solutions.facemesh.FaceMesh;
import com.google.mediapipe.solutions.facemesh.FaceMeshOptions;
import com.google.mediapipe.solutions.facemesh.FaceMeshResult;
import java.io.IOException;
import java.io.InputStream;
/** Main activity of MediaPipe FaceMesh app. */
/** Main activity of MediaPipe Face Mesh app. */
public class MainActivity extends AppCompatActivity {
private static final String TAG = "MainActivity";
@@ -57,12 +61,14 @@ public class MainActivity extends AppCompatActivity {
private ActivityResultLauncher<Intent> videoGetter;
// Live camera demo UI and camera components.
private CameraInput cameraInput;
private SolutionGlSurfaceView<FaceMeshResult> glSurfaceView;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
// TODO: Add a toggle to switch between the original face mesh and attention mesh.
setupStaticImageDemoUiComponents();
setupVideoDemoUiComponents();
setupLiveDemoUiComponents();
@@ -111,6 +117,35 @@ public class MainActivity extends AppCompatActivity {
} catch (IOException e) {
Log.e(TAG, "Bitmap reading error:" + e);
}
try {
InputStream imageData =
this.getContentResolver().openInputStream(resultIntent.getData());
int orientation =
new ExifInterface(imageData)
.getAttributeInt(
ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
if (orientation != ExifInterface.ORIENTATION_NORMAL) {
Matrix matrix = new Matrix();
switch (orientation) {
case ExifInterface.ORIENTATION_ROTATE_90:
matrix.postRotate(90);
break;
case ExifInterface.ORIENTATION_ROTATE_180:
matrix.postRotate(180);
break;
case ExifInterface.ORIENTATION_ROTATE_270:
matrix.postRotate(270);
break;
default:
matrix.postRotate(0);
}
bitmap =
Bitmap.createBitmap(
bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
}
} catch (IOException e) {
Log.e(TAG, "Bitmap rotation error:" + e);
}
if (bitmap != null) {
facemesh.send(bitmap);
}
@@ -132,26 +167,27 @@ public class MainActivity extends AppCompatActivity {
imageView = new FaceMeshResultImageView(this);
}
/** The core MediaPipe FaceMesh setup workflow for its static image mode. */
/** Sets up core workflow for static image mode. */
private void setupStaticImageModePipeline() {
this.inputSource = InputSource.IMAGE;
// Initializes a new MediaPipe FaceMesh instance in the static image mode.
// Initializes a new MediaPipe Face Mesh solution instance in the static image mode.
facemesh =
new FaceMesh(
this,
FaceMeshOptions.builder()
.setMode(FaceMeshOptions.STATIC_IMAGE_MODE)
.setStaticImageMode(true)
.setRefineLandmarks(true)
.setRunOnGpu(RUN_ON_GPU)
.build());
// Connects MediaPipe FaceMesh to the user-defined FaceMeshResultImageView.
// Connects MediaPipe Face Mesh solution to the user-defined FaceMeshResultImageView.
facemesh.setResultListener(
faceMeshResult -> {
logNoseLandmark(faceMeshResult, /*showPixelValues=*/ true);
imageView.setFaceMeshResult(faceMeshResult);
runOnUiThread(() -> imageView.update());
});
facemesh.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
facemesh.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
// Updates the preview layout.
FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
@@ -207,25 +243,24 @@ public class MainActivity extends AppCompatActivity {
});
}
/** The core MediaPipe FaceMesh setup workflow for its streaming mode. */
/** Sets up core workflow for streaming mode. */
private void setupStreamingModePipeline(InputSource inputSource) {
this.inputSource = inputSource;
// Initializes a new MediaPipe FaceMesh instance in the streaming mode.
// Initializes a new MediaPipe Face Mesh solution instance in the streaming mode.
facemesh =
new FaceMesh(
this,
FaceMeshOptions.builder()
.setMode(FaceMeshOptions.STREAMING_MODE)
.setStaticImageMode(false)
.setRefineLandmarks(true)
.setRunOnGpu(RUN_ON_GPU)
.build());
facemesh.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
facemesh.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
if (inputSource == InputSource.CAMERA) {
// Initializes a new CameraInput instance and connects it to MediaPipe FaceMesh.
cameraInput = new CameraInput(this);
cameraInput.setNewFrameListener(textureFrame -> facemesh.send(textureFrame));
} else if (inputSource == InputSource.VIDEO) {
// Initializes a new VideoInput instance and connects it to MediaPipe FaceMesh.
videoInput = new VideoInput(this);
videoInput.setNewFrameListener(textureFrame -> facemesh.send(textureFrame));
}
@@ -295,13 +330,13 @@ public class MainActivity extends AppCompatActivity {
Log.i(
TAG,
String.format(
"MediaPipe FaceMesh nose coordinates (pixel values): x=%f, y=%f",
"MediaPipe Face Mesh nose coordinates (pixel values): x=%f, y=%f",
noseLandmark.getX() * width, noseLandmark.getY() * height));
} else {
Log.i(
TAG,
String.format(
"MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
"MediaPipe Face Mesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
noseLandmark.getX(), noseLandmark.getY()));
}
}
mediapipe/examples/android/solutions/hands/build.gradle
+4 -3
View File
@@ -31,15 +31,16 @@ dependencies {
implementation 'androidx.appcompat:appcompat:1.3.0'
implementation 'com.google.android.material:material:1.3.0'
implementation 'androidx.constraintlayout:constraintlayout:2.0.4'
implementation 'androidx.exifinterface:exifinterface:1.3.3'
testImplementation 'junit:junit:4.+'
androidTestImplementation 'androidx.test.ext:junit:1.1.2'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
// MediaPipe hands solution API and solution-core.
// MediaPipe Hands Solution components.
implementation 'com.google.mediapipe:solution-core:latest.release'
implementation 'com.google.mediapipe:hands:latest.release'
// MediaPipe deps
implementation 'com.google.flogger:flogger:latest.release'
implementation 'com.google.flogger:flogger-system-backend:latest.release'
implementation 'com.google.flogger:flogger:0.6'
implementation 'com.google.flogger:flogger-system-backend:0.6'
implementation 'com.google.guava:guava:27.0.1-android'
implementation 'com.google.protobuf:protobuf-java:3.11.4'
// CameraX core library
mediapipe/examples/android/solutions/hands/src/main/BUILD
+1
View File
@@ -38,6 +38,7 @@ android_binary(
"//third_party:opencv",
"@maven//:androidx_activity_activity",
"@maven//:androidx_concurrent_concurrent_futures",
"@maven//:androidx_exifinterface_exifinterface",
"@maven//:androidx_fragment_fragment",
"@maven//:com_google_guava_guava",
],
mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java
+9 -34
View File
@@ -15,9 +15,7 @@
package com.google.mediapipe.examples.hands;
import android.opengl.GLES20;
import android.opengl.Matrix;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
import com.google.mediapipe.solutioncore.ResultGlBoundary;
import com.google.mediapipe.solutioncore.ResultGlRenderer;
import com.google.mediapipe.solutions.hands.Hands;
import com.google.mediapipe.solutions.hands.HandsResult;
@@ -26,16 +24,16 @@ import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.util.List;
/** A custom implementation of {@link ResultGlRenderer} to render MediaPope Hands results. */
/** A custom implementation of {@link ResultGlRenderer} to render {@link HandsResult}. */
public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
private static final String TAG = "HandsResultGlRenderer";
private static final float CONNECTION_THICKNESS = 20.0f;
private static final String VERTEX_SHADER =
"uniform mat4 uTransformMatrix;\n"
"uniform mat4 uProjectionMatrix;\n"
+ "attribute vec4 vPosition;\n"
+ "void main() {\n"
+ " gl_Position = uTransformMatrix * vPosition;\n"
+ " gl_Position = uProjectionMatrix * vPosition;\n"
+ "}";
private static final String FRAGMENT_SHADER =
"precision mediump float;\n"
@@ -44,8 +42,7 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
+ "}";
private int program;
private int positionHandle;
private int transformMatrixHandle;
private final float[] transformMatrix = new float[16];
private int projectionMatrixHandle;
private int loadShader(int type, String shaderCode) {
int shader = GLES20.glCreateShader(type);
@@ -63,27 +60,16 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
GLES20.glAttachShader(program, fragmentShader);
GLES20.glLinkProgram(program);
positionHandle = GLES20.glGetAttribLocation(program, "vPosition");
transformMatrixHandle = GLES20.glGetUniformLocation(program, "uTransformMatrix");
projectionMatrixHandle = GLES20.glGetUniformLocation(program, "uProjectionMatrix");
}
@Override
public void renderResult(HandsResult result, ResultGlBoundary boundary) {
public void renderResult(HandsResult result, float[] projectionMatrix) {
if (result == null) {
return;
}
GLES20.glUseProgram(program);
// Sets the transform matrix to align the result rendering with the scaled output texture.
// Also flips the rendering vertically since OpenGL assumes the coordinate origin is at the
// bottom-left corner, whereas MediaPipe landmark data assumes the coordinate origin is at the
// top-left corner.
Matrix.setIdentityM(transformMatrix, 0);
Matrix.scaleM(
transformMatrix,
0,
2 / (boundary.right() - boundary.left()),
-2 / (boundary.top() - boundary.bottom()),
1.0f);
GLES20.glUniformMatrix4fv(transformMatrixHandle, 1, false, transformMatrix, 0);
GLES20.glUniformMatrix4fv(projectionMatrixHandle, 1, false, projectionMatrix, 0);
GLES20.glLineWidth(CONNECTION_THICKNESS);
int numHands = result.multiHandLandmarks().size();
@@ -93,7 +79,7 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
}
/**
* Calls this to delete the shader program.
* Deletes the shader program.
*
* <p>This is only necessary if one wants to release the program while keeping the context around.
*/
@@ -101,16 +87,11 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
GLES20.glDeleteProgram(program);
}
// TODO: Better hand landmark and hand connection drawing.
private void drawLandmarks(List<NormalizedLandmark> handLandmarkList) {
for (Hands.Connection c : Hands.HAND_CONNECTIONS) {
float[] vertex = new float[4];
NormalizedLandmark start = handLandmarkList.get(c.start());
vertex[0] = normalizedLandmarkValue(start.getX());
vertex[1] = normalizedLandmarkValue(start.getY());
NormalizedLandmark end = handLandmarkList.get(c.end());
vertex[2] = normalizedLandmarkValue(end.getX());
vertex[3] = normalizedLandmarkValue(end.getY());
float[] vertex = {start.getX(), start.getY(), end.getX(), end.getY()};
FloatBuffer vertexBuffer =
ByteBuffer.allocateDirect(vertex.length * 4)
.order(ByteOrder.nativeOrder())
@@ -122,10 +103,4 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
GLES20.glDrawArrays(GLES20.GL_LINES, 0, 2);
}
}
// Normalizes the value from the landmark value range:[0, 1] to the standard OpenGL coordinate
// value range: [-1, 1].
private float normalizedLandmarkValue(float value) {
return value * 2 - 1;
}
}
mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java
+2 -3
View File
@@ -27,7 +27,7 @@ import com.google.mediapipe.solutions.hands.Hands;
import com.google.mediapipe.solutions.hands.HandsResult;
import java.util.List;
/** An ImageView implementation for displaying MediaPipe Hands results. */
/** An ImageView implementation for displaying {@link HandsResult}. */
public class HandsResultImageView extends AppCompatImageView {
private static final String TAG = "HandsResultImageView";
@@ -66,7 +66,7 @@ public class HandsResultImageView extends AppCompatImageView {
}
}
/** Updates the image view with the latest hands result. */
/** Updates the image view with the latest {@link HandsResult}. */
public void update() {
postInvalidate();
if (latest != null) {
@@ -74,7 +74,6 @@ public class HandsResultImageView extends AppCompatImageView {
}
}
// TODO: Better hand landmark and hand connection drawing.
private void drawLandmarksOnCanvas(
List<NormalizedLandmark> handLandmarkList, Canvas canvas, int width, int height) {
// Draw connections.
mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/MainActivity.java
+41 -9
View File
@@ -16,6 +16,7 @@ package com.google.mediapipe.examples.hands;
import android.content.Intent;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.os.Bundle;
import android.provider.MediaStore;
import androidx.appcompat.app.AppCompatActivity;
@@ -25,6 +26,8 @@ import android.widget.Button;
import android.widget.FrameLayout;
import androidx.activity.result.ActivityResultLauncher;
import androidx.activity.result.contract.ActivityResultContracts;
import androidx.exifinterface.media.ExifInterface;
// ContentResolver dependency
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
import com.google.mediapipe.solutioncore.CameraInput;
import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
@@ -34,6 +37,7 @@ import com.google.mediapipe.solutions.hands.Hands;
import com.google.mediapipe.solutions.hands.HandsOptions;
import com.google.mediapipe.solutions.hands.HandsResult;
import java.io.IOException;
import java.io.InputStream;
/** Main activity of MediaPipe Hands app. */
public class MainActivity extends AppCompatActivity {
@@ -59,6 +63,7 @@ public class MainActivity extends AppCompatActivity {
private ActivityResultLauncher<Intent> videoGetter;
// Live camera demo UI and camera components.
private CameraInput cameraInput;
private SolutionGlSurfaceView<HandsResult> glSurfaceView;
@Override
@@ -113,6 +118,35 @@ public class MainActivity extends AppCompatActivity {
} catch (IOException e) {
Log.e(TAG, "Bitmap reading error:" + e);
}
try {
InputStream imageData =
this.getContentResolver().openInputStream(resultIntent.getData());
int orientation =
new ExifInterface(imageData)
.getAttributeInt(
ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
if (orientation != ExifInterface.ORIENTATION_NORMAL) {
Matrix matrix = new Matrix();
switch (orientation) {
case ExifInterface.ORIENTATION_ROTATE_90:
matrix.postRotate(90);
break;
case ExifInterface.ORIENTATION_ROTATE_180:
matrix.postRotate(180);
break;
case ExifInterface.ORIENTATION_ROTATE_270:
matrix.postRotate(270);
break;
default:
matrix.postRotate(0);
}
bitmap =
Bitmap.createBitmap(
bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
}
} catch (IOException e) {
Log.e(TAG, "Bitmap rotation error:" + e);
}
if (bitmap != null) {
hands.send(bitmap);
}
@@ -134,20 +168,20 @@ public class MainActivity extends AppCompatActivity {
imageView = new HandsResultImageView(this);
}
/** The core MediaPipe Hands setup workflow for its static image mode. */
/** Sets up core workflow for static image mode. */
private void setupStaticImageModePipeline() {
this.inputSource = InputSource.IMAGE;
// Initializes a new MediaPipe Hands instance in the static image mode.
// Initializes a new MediaPipe Hands solution instance in the static image mode.
hands =
new Hands(
this,
HandsOptions.builder()
.setMode(HandsOptions.STATIC_IMAGE_MODE)
.setStaticImageMode(true)
.setMaxNumHands(1)
.setRunOnGpu(RUN_ON_GPU)
.build());
// Connects MediaPipe Hands to the user-defined HandsResultImageView.
// Connects MediaPipe Hands solution to the user-defined HandsResultImageView.
hands.setResultListener(
handsResult -> {
logWristLandmark(handsResult, /*showPixelValues=*/ true);
@@ -210,26 +244,24 @@ public class MainActivity extends AppCompatActivity {
});
}
/** The core MediaPipe Hands setup workflow for its streaming mode. */
/** Sets up core workflow for streaming mode. */
private void setupStreamingModePipeline(InputSource inputSource) {
this.inputSource = inputSource;
// Initializes a new MediaPipe Hands instance in the streaming mode.
// Initializes a new MediaPipe Hands solution instance in the streaming mode.
hands =
new Hands(
this,
HandsOptions.builder()
.setMode(HandsOptions.STREAMING_MODE)
.setStaticImageMode(false)
.setMaxNumHands(1)
.setRunOnGpu(RUN_ON_GPU)
.build());
hands.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
if (inputSource == InputSource.CAMERA) {
// Initializes a new CameraInput instance and connects it to MediaPipe Hands.
cameraInput = new CameraInput(this);
cameraInput.setNewFrameListener(textureFrame -> hands.send(textureFrame));
} else if (inputSource == InputSource.VIDEO) {
// Initializes a new VideoInput instance and connects it to MediaPipe Hands.
videoInput = new VideoInput(this);
videoInput.setNewFrameListener(textureFrame -> hands.send(textureFrame));
}
mediapipe/examples/android/solutions/settings.gradle
+2 -1
View File
@@ -1,3 +1,4 @@
rootProject.name = "mediapipe-solutions-examples"
include ':hands'
include ':facedetection'
include ':facemesh'
include ':hands'
mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD
+1 -1
View File
@@ -37,7 +37,7 @@ android_binary(
srcs = glob(["*.java"]),
assets = [
"//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb",
"//mediapipe/modules/face_landmark:face_landmark.tflite",
"//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite",
"//mediapipe/modules/face_detection:face_detection_short_range.tflite",
],
assets_dir = "",
mediapipe/examples/ios/facemeshgpu/BUILD
+1 -1
View File
@@ -63,7 +63,7 @@ objc_library(
data = [
"//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb",
"//mediapipe/modules/face_detection:face_detection_short_range.tflite",
"//mediapipe/modules/face_landmark:face_landmark.tflite",
"//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite",
],
deps = [
"//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary",