Project import generated by Copybara.

GitOrigin-RevId: 4cee4a2c2317fb190680c17e31ebbb03bb73b71c
2020-09-17 11:09:17 -04:00
parent 1db91b550a
commit a908d668c7
142 changed files with 40878 additions and 574 deletions
@@ -0,0 +1,67 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "libmediapipe_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/graphs/face_effect:face_effect_gpu_deps",
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+    ],
+)
+
+cc_library(
+    name = "mediapipe_jni_lib",
+    srcs = [":libmediapipe_jni.so"],
+    alwayslink = 1,
+)
+
+android_binary(
+    name = "faceeffect",
+    srcs = glob(["*.java"]),
+    assets = [
+        "//mediapipe/graphs/face_effect/data:facepaint.pngblob",
+        "//mediapipe/graphs/face_effect/data:glasses.binarypb",
+        "//mediapipe/graphs/face_effect/data:glasses.pngblob",
+        "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+        "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+    ],
+    assets_dir = "",
+    manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",
+    manifest_values = {
+        "applicationId": "com.google.mediapipe.apps.faceeffect",
+        "appName": "Face Effect",
+        "mainActivity": ".MainActivity",
+        "cameraFacingFront": "True",
+        "binaryGraphName": "face_effect_gpu.binarypb",
+        "inputVideoStreamName": "input_video",
+        "outputVideoStreamName": "output_video",
+        "flipFramesVertically": "True",
+    },
+    multidex = "native",
+    deps = [
+        ":mediapipe_jni_lib",
+        "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib",
+        "//mediapipe/framework/formats:matrix_data_java_proto_lite",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_java_proto_lite",
+    ],
+)
@@ -0,0 +1,176 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.apps.faceeffect;
+
+import android.graphics.Color;
+import android.os.Bundle;
+import android.util.Log;
+import android.view.GestureDetector;
+import android.view.Gravity;
+import android.view.MotionEvent;
+import android.view.View;
+import android.view.ViewGroup;
+import android.view.ViewGroup.LayoutParams;
+import android.widget.RelativeLayout;
+import android.widget.TextView;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.framework.PacketGetter;
+import com.google.mediapipe.modules.facegeometry.FaceGeometryProto.FaceGeometry;
+import com.google.mediapipe.formats.proto.MatrixDataProto.MatrixData;
+import java.util.List;
+
+/** Main activity of MediaPipe face mesh app. */
+public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
+  private static final String TAG = "MainActivity";
+
+  // Stream names.
+  private static final String IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME =
+      "is_facepaint_effect_selected";
+  private static final String OUTPUT_FACE_GEOMETRY_STREAM_NAME = "multi_face_geometry";
+
+  private static final String EFFECT_SWITCHING_HINT_TEXT = "Tap to switch between effects!";
+
+  private static final int MATRIX_TRANSLATION_Z_INDEX = 14;
+
+  private final Object isFacepaintEffectSelectedLock = new Object();
+  private boolean isFacepaintEffectSelected;
+
+  private View effectSwitchingHintView;
+  private GestureDetector tapGestureDetector;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+
+    // Add an effect switching hint view to the preview layout.
+    effectSwitchingHintView = createEffectSwitchingHintView();
+    effectSwitchingHintView.setVisibility(View.INVISIBLE);
+    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
+    viewGroup.addView(effectSwitchingHintView);
+
+    // By default, render the glasses effect.
+    isFacepaintEffectSelected = false;
+
+    // This callback demonstrates how the output face geometry packet can be obtained and used
+    // in an Android app. As an example, the Z-translation component of the face pose transform
+    // matrix is logged for each face being equal to the approximate distance away from the camera
+    // in centimeters.
+    processor.addPacketCallback(
+        OUTPUT_FACE_GEOMETRY_STREAM_NAME,
+        (packet) -> {
+          effectSwitchingHintView.post(
+              new Runnable() {
+                @Override
+                public void run() {
+                  effectSwitchingHintView.setVisibility(View.VISIBLE);
+                }
+              });
+
+          Log.d(TAG, "Received a multi face geometry packet.");
+          List<FaceGeometry> multiFaceGeometry =
+              PacketGetter.getProtoVector(packet, FaceGeometry.parser());
+
+          StringBuilder approxDistanceAwayFromCameraLogMessage = new StringBuilder();
+          for (FaceGeometry faceGeometry : multiFaceGeometry) {
+            if (approxDistanceAwayFromCameraLogMessage.length() > 0) {
+              approxDistanceAwayFromCameraLogMessage.append(' ');
+            }
+            MatrixData poseTransformMatrix = faceGeometry.getPoseTransformMatrix();
+            approxDistanceAwayFromCameraLogMessage.append(
+                -poseTransformMatrix.getPackedData(MATRIX_TRANSLATION_Z_INDEX));
+          }
+
+          Log.d(
+              TAG,
+              "[TS:"
+                  + packet.getTimestamp()
+                  + "] size = "
+                  + multiFaceGeometry.size()
+                  + "; approx. distance away from camera in cm for faces = ["
+                  + approxDistanceAwayFromCameraLogMessage
+                  + "]");
+        });
+
+    // Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
+    // packet to indicate which effect should be rendered on this frame.
+    processor.setOnWillAddFrameListener(
+        (timestamp) -> {
+          Packet isFacepaintEffectSelectedPacket = null;
+          try {
+            synchronized (isFacepaintEffectSelectedLock) {
+              isFacepaintEffectSelectedPacket =
+                  processor.getPacketCreator().createBool(isFacepaintEffectSelected);
+            }
+
+            processor
+                .getGraph()
+                .addPacketToInputStream(
+                    IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME,
+                    isFacepaintEffectSelectedPacket,
+                    timestamp);
+          } catch (RuntimeException e) {
+            Log.e(
+                TAG,
+                "Exception while adding packet to input stream while switching effects: " + e);
+          } finally {
+            if (isFacepaintEffectSelectedPacket != null) {
+              isFacepaintEffectSelectedPacket.release();
+            }
+          }
+        });
+
+    // We use the tap gesture detector to switch between face effects. This allows users to try
+    // multiple pre-bundled face effects without a need to recompile the app.
+    tapGestureDetector =
+        new GestureDetector(
+            this,
+            new GestureDetector.SimpleOnGestureListener() {
+              @Override
+              public void onLongPress(MotionEvent event) {
+                switchEffect();
+              }
+
+              @Override
+              public boolean onSingleTapUp(MotionEvent event) {
+                switchEffect();
+                return true;
+              }
+
+              private void switchEffect() {
+                synchronized (isFacepaintEffectSelectedLock) {
+                  isFacepaintEffectSelected = !isFacepaintEffectSelected;
+                }
+              }
+            });
+  }
+
+  @Override
+  public boolean onTouchEvent(MotionEvent event) {
+    return tapGestureDetector.onTouchEvent(event);
+  }
+
+  private View createEffectSwitchingHintView() {
+    TextView effectSwitchingHintView = new TextView(getApplicationContext());
+    effectSwitchingHintView.setLayoutParams(
+        new RelativeLayout.LayoutParams(LayoutParams.FILL_PARENT, LayoutParams.FILL_PARENT));
+    effectSwitchingHintView.setText(EFFECT_SWITCHING_HINT_TEXT);
+    effectSwitchingHintView.setGravity(Gravity.CENTER_HORIZONTAL | Gravity.BOTTOM);
+    effectSwitchingHintView.setPadding(0, 0, 0, 480);
+    effectSwitchingHintView.setTextColor(Color.parseColor("#ffffff"));
+    effectSwitchingHintView.setTextSize((float) 24);
+
+    return effectSwitchingHintView;
+  }
+}
@@ -35,7 +35,7 @@ constexpr char kDetectedBorders[] = "BORDERS";
 constexpr char kCropRect[] = "CROP_RECT";
 // Field-of-view (degrees) of the camera's x-axis (width).
 // TODO: Parameterize FOV based on camera specs.
-constexpr float kWidthFieldOfView = 60;
+constexpr float kFieldOfView = 60;

 namespace mediapipe {
 namespace autoflip {
@@ -256,13 +256,13 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
  if (!initialized_) {
    path_solver_height_ = std::make_unique<KinematicPathSolver>(
        options_.kinematic_options_zoom(), 0, frame_height_,
-        static_cast<float>(frame_width_) / kWidthFieldOfView);
+        static_cast<float>(frame_height_) / kFieldOfView);
    path_solver_width_ = std::make_unique<KinematicPathSolver>(
        options_.kinematic_options_pan(), 0, frame_width_,
-        static_cast<float>(frame_width_) / kWidthFieldOfView);
+        static_cast<float>(frame_width_) / kFieldOfView);
    path_solver_offset_ = std::make_unique<KinematicPathSolver>(
        options_.kinematic_options_tilt(), 0, frame_height_,
-        static_cast<float>(frame_width_) / kWidthFieldOfView);
+        static_cast<float>(frame_height_) / kFieldOfView);
    max_frame_value_ = 1.0;
    target_aspect_ = frame_width_ / static_cast<float>(frame_height_);
    // If target size is set and wider than input aspect, make sure to always
@@ -339,15 +339,24 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
    offset_y = last_measured_y_offset_;
  }

-  // Compute smoothed camera paths.
+  // Compute smoothed zoom camera path.
  MP_RETURN_IF_ERROR(path_solver_height_->AddObservation(
      height, cc->InputTimestamp().Microseconds()));
+  int path_height;
+  MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
+  int path_width = path_height * target_aspect_;
+
+  // Update pixel-per-degree value for pan/tilt.
+  MP_RETURN_IF_ERROR(path_solver_width_->UpdatePixelsPerDegree(
+      static_cast<float>(path_width) / kFieldOfView));
+  MP_RETURN_IF_ERROR(path_solver_offset_->UpdatePixelsPerDegree(
+      static_cast<float>(path_height) / kFieldOfView));
+
+  // Compute smoothed pan/tilt paths.
  MP_RETURN_IF_ERROR(path_solver_width_->AddObservation(
      offset_x, cc->InputTimestamp().Microseconds()));
  MP_RETURN_IF_ERROR(path_solver_offset_->AddObservation(
      offset_y, cc->InputTimestamp().Microseconds()));
-  int path_height;
-  MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
  int path_offset_x;
  MP_RETURN_IF_ERROR(path_solver_width_->GetState(&path_offset_x));
  int path_offset_y;
@@ -359,7 +368,7 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
  } else if (path_offset_y + ceil(path_height / 2.0) > frame_height_) {
    path_offset_y = frame_height_ - ceil(path_height / 2.0);
  }
-  int path_width = path_height * target_aspect_;
+
  if (path_offset_x - ceil(path_width / 2.0) < 0) {
    path_offset_x = ceil(path_width / 2.0);
  } else if (path_offset_x + ceil(path_width / 2.0) > frame_width_) {
@@ -174,15 +174,15 @@ TEST(ContentZoomingCalculatorTest, PanConfig) {
      ContentZoomingCalculatorOptions::ext);
  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(0.0);
  options->mutable_kinematic_options_pan()->set_update_rate_seconds(2);
-  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
-  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
+  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
  auto runner = ::absl::make_unique<CalculatorRunner>(config);
  AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
  AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
  MP_ASSERT_OK(runner->Run());
  CheckCropRect(450, 550, 111, 111, 0,
                runner->Outputs().Tag("CROP_RECT").packets);
-  CheckCropRect(488, 550, 111, 111, 1,
+  CheckCropRect(483, 550, 111, 111, 1,
                runner->Outputs().Tag("CROP_RECT").packets);
 }

@@ -190,17 +190,17 @@ TEST(ContentZoomingCalculatorTest, TiltConfig) {
  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
  auto* options = config.mutable_options()->MutableExtension(
      ContentZoomingCalculatorOptions::ext);
-  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0);
  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(0.0);
  options->mutable_kinematic_options_tilt()->set_update_rate_seconds(2);
-  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
  auto runner = ::absl::make_unique<CalculatorRunner>(config);
  AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
  AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
  MP_ASSERT_OK(runner->Run());
  CheckCropRect(450, 550, 111, 111, 0,
                runner->Outputs().Tag("CROP_RECT").packets);
-  CheckCropRect(450, 588, 111, 111, 1,
+  CheckCropRect(450, 583, 111, 111, 1,
                runner->Outputs().Tag("CROP_RECT").packets);
 }

@@ -208,8 +208,8 @@ TEST(ContentZoomingCalculatorTest, ZoomConfig) {
  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
  auto* options = config.mutable_options()->MutableExtension(
      ContentZoomingCalculatorOptions::ext);
-  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
-  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0);
+  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(0.0);
  options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2);
  auto runner = ::absl::make_unique<CalculatorRunner>(config);
@@ -87,5 +87,13 @@ namespace autoflip {
  return ::mediapipe::OkStatus();
 }

+::mediapipe::Status KinematicPathSolver::UpdatePixelsPerDegree(
+    const float pixels_per_degree) {
+  RET_CHECK_GT(pixels_per_degree_, 0)
+      << "pixels_per_degree must be larger than 0.";
+  pixels_per_degree_ = pixels_per_degree;
+  return ::mediapipe::OkStatus();
+}
+
 }  // namespace autoflip
 }  // namespace mediapipe
@@ -46,6 +46,8 @@ class KinematicPathSolver {
  ::mediapipe::Status UpdatePrediction(const int64 time_us);
  // Get the state at a time.
  ::mediapipe::Status GetState(int* position);
+  // Update PixelPerDegree value.
+  ::mediapipe::Status UpdatePixelsPerDegree(const float pixels_per_degree);

 private:
  // Tuning options.
@@ -207,6 +207,28 @@ TEST(KinematicPathSolverTest, PassMaxVelocity) {
  EXPECT_EQ(state, 600);
 }

+TEST(KinematicPathSolverTest, PassDegPerPxChange) {
+  KinematicOptions options;
+  // Set min motion to 2deg
+  options.set_min_motion_to_reframe(2.0);
+  options.set_update_rate(1);
+  options.set_max_velocity(1000);
+  // Set degrees / pixel to 16.6
+  KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
+  int state;
+  MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
+  // Move target by 20px / 16.6 = 1.2deg
+  MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
+  MP_ASSERT_OK(solver.GetState(&state));
+  // Expect cam to not move.
+  EXPECT_EQ(state, 500);
+  MP_ASSERT_OK(solver.UpdatePixelsPerDegree(500.0 / kWidthFieldOfView));
+  MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2));
+  MP_ASSERT_OK(solver.GetState(&state));
+  // Expect cam to move.
+  EXPECT_EQ(state, 516);
+}
+
 }  // namespace
 }  // namespace autoflip
 }  // namespace mediapipe
@@ -1,5 +1,5 @@
 # MediaPipe graph that performs face detection with TensorFlow Lite on CPU.  Model paths setup for web use.
-# TODO: parameterize input paths to support desktop use.
+# TODO: parameterize input paths to support desktop use, for web only.
 input_stream: "VIDEO:input_video"
 output_stream: "DETECTIONS:output_detections"

@@ -37,7 +37,7 @@ node {
  output_stream: "TENSORS:detection_tensors"
  options: {
    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
-      model_path: "mediapipe/models/face_detection_front.tflite"
+      model_path: "face_detection_front.tflite"
    }
  }
 }
@@ -118,7 +118,7 @@ node {
  output_stream: "labeled_detections"
  options: {
    [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
-      label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
+      label_map_path: "face_detection_front_labelmap.txt"
    }
  }
 }
@@ -0,0 +1,21 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow *window;
+
+@end
@@ -0,0 +1,59 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+
+- (BOOL)application:(UIApplication *)application
+    didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+  // Override point for customization after application launch.
+  return YES;
+}
+
+- (void)applicationWillResignActive:(UIApplication *)application {
+  // Sent when the application is about to move from active to inactive state. This can occur for
+  // certain types of temporary interruptions (such as an incoming phone call or SMS message) or
+  // when the user quits the application and it begins the transition to the background state. Use
+  // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering
+  // callbacks. Games should use this method to pause the game.
+}
+
+- (void)applicationDidEnterBackground:(UIApplication *)application {
+  // Use this method to release shared resources, save user data, invalidate timers, and store
+  // enough application state information to restore your application to its current state in case
+  // it is terminated later. If your application supports background execution, this method is
+  // called instead of applicationWillTerminate: when the user quits.
+}
+
+- (void)applicationWillEnterForeground:(UIApplication *)application {
+  // Called as part of the transition from the background to the active state; here you can undo
+  // many of the changes made on entering the background.
+}
+
+- (void)applicationDidBecomeActive:(UIApplication *)application {
+  // Restart any tasks that were paused (or not yet started) while the application was inactive. If
+  // the application was previously in the background, optionally refresh the user interface.
+}
+
+- (void)applicationWillTerminate:(UIApplication *)application {
+  // Called when the application is about to terminate. Save data if appropriate. See also
+  // applicationDidEnterBackground:.
+}
+
+@end
@@ -0,0 +1,92 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "@build_bazel_rules_apple//apple:ios.bzl",
+    "ios_application",
+)
+load(
+    "//mediapipe/examples/ios:bundle_id.bzl",
+    "BUNDLE_ID_PREFIX",
+    "example_provisioning",
+)
+
+licenses(["notice"])
+
+MIN_IOS_VERSION = "10.0"
+
+alias(
+    name = "faceeffect",
+    actual = "FaceEffectApp",
+)
+
+ios_application(
+    name = "FaceEffectApp",
+    app_icons = ["//mediapipe/examples/ios/common:AppIcon"],
+    bundle_id = BUNDLE_ID_PREFIX + ".FaceMeshGpu",
+    families = [
+        "iphone",
+        "ipad",
+    ],
+    infoplists = ["Info.plist"],
+    minimum_os_version = MIN_IOS_VERSION,
+    provisioning_profile = example_provisioning(),
+    deps = [
+        ":FaceEffectAppLibrary",
+        "@ios_opencv//:OpencvFramework",
+    ],
+)
+
+objc_library(
+    name = "FaceEffectAppLibrary",
+    srcs = [
+        "AppDelegate.m",
+        "FaceEffectViewController.mm",
+        "main.m",
+    ],
+    hdrs = [
+        "AppDelegate.h",
+        "FaceEffectViewController.h",
+    ],
+    data = [
+        "Base.lproj/LaunchScreen.storyboard",
+        "Base.lproj/Main.storyboard",
+        "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
+        "//mediapipe/graphs/face_effect/data:facepaint.pngblob",
+        "//mediapipe/graphs/face_effect/data:glasses.binarypb",
+        "//mediapipe/graphs/face_effect/data:glasses.pngblob",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+        "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+    ],
+    sdk_frameworks = [
+        "AVFoundation",
+        "CoreGraphics",
+        "CoreMedia",
+        "UIKit",
+    ],
+    deps = [
+        "//mediapipe/objc:mediapipe_framework_ios",
+        "//mediapipe/objc:mediapipe_input_sources_ios",
+        "//mediapipe/objc:mediapipe_layer_renderer",
+    ] + select({
+        "//mediapipe:ios_i386": [],
+        "//mediapipe:ios_x86_64": [],
+        "//conditions:default": [
+            "//mediapipe/framework/formats:matrix_data_cc_proto",
+            "//mediapipe/graphs/face_effect:face_effect_gpu_deps",
+            "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        ],
+    }),
+)
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="16096" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina5_5" orientation="portrait" appearance="light"/>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="16086"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="FaceEffectViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="414" height="736"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
+                                <rect key="frame" x="0.0" y="20" width="414" height="716"/>
+                                <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                                <subviews>
+                                    <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
+                                        <rect key="frame" x="76" y="283" width="260" height="151"/>
+                                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
+                                        <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                        <color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <nil key="highlightedColor"/>
+                                    </label>
+                                    <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Tap to switch between effects!" textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="b2C-zd-Zba" userLabel="Effect Switching Label">
+                                        <rect key="frame" x="77" y="488" width="260" height="151"/>
+                                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
+                                        <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                        <color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <nil key="highlightedColor"/>
+                                    </label>
+                                </subviews>
+                                <color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                <accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
+                                    <bool key="isElement" value="YES"/>
+                                </accessibility>
+                            </view>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                    <connections>
+                        <outlet property="_effectSwitchingHintLabel" destination="b2C-zd-Zba" id="Uvx-2n-l74"/>
+                        <outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
+                        <outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="48.799999999999997" y="20.239880059970016"/>
+        </scene>
+    </scenes>
+</document>
@@ -0,0 +1,19 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface FaceEffectViewController : UIViewController
+
+@end
@@ -0,0 +1,254 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "FaceEffectViewController.h"
+
+#import "mediapipe/objc/MPPCameraInputSource.h"
+#import "mediapipe/objc/MPPGraph.h"
+#import "mediapipe/objc/MPPLayerRenderer.h"
+
+#include <utility>
+
+#include "mediapipe/framework/formats/matrix_data.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+
+static NSString* const kGraphName = @"face_effect_gpu";
+
+static const char* kInputStream = "input_video";
+static const char* kIsFacepaintEffectSelectedInputStream = "is_facepaint_effect_selected";
+static const char* kOutputStream = "output_video";
+static const char* kMultiFaceGeometryStream = "multi_face_geometry";
+static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
+
+static const int kMatrixTranslationZIndex = 14;
+
+@interface FaceEffectViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
+
+// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and
+// sent video frames on _videoQueue.
+@property(nonatomic) MPPGraph* graph;
+
+@end
+
+@implementation FaceEffectViewController {
+  /// Handle tap gestures.
+  UITapGestureRecognizer* _tapGestureRecognizer;
+  BOOL _isFacepaintEffectSelected;
+
+  /// Handles camera access via AVCaptureSession library.
+  MPPCameraInputSource* _cameraSource;
+
+  /// Inform the user when camera is unavailable.
+  IBOutlet UILabel* _noCameraLabel;
+  /// Inform the user about how to switch between effects.
+  UILabel* _effectSwitchingHintLabel;
+  /// Display the camera preview frames.
+  IBOutlet UIView* _liveView;
+  /// Render frames in a layer.
+  MPPLayerRenderer* _renderer;
+
+  /// Process camera frames on this queue.
+  dispatch_queue_t _videoQueue;
+}
+
+#pragma mark - Cleanup methods
+
+- (void)dealloc {
+  self.graph.delegate = nil;
+  [self.graph cancel];
+  // Ignore errors since we're cleaning up.
+  [self.graph closeAllInputStreamsWithError:nil];
+  [self.graph waitUntilDoneWithError:nil];
+}
+
+#pragma mark - MediaPipe graph methods
+
+ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
+  // Load the graph config resource.
+  NSError* configLoadError = nil;
+  NSBundle* bundle = [NSBundle bundleForClass:[self class]];
+  if (!resource || resource.length == 0) {
+    return nil;
+  }
+  NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
+  NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
+  if (!data) {
+    NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError);
+    return nil;
+  }
+
+  // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object.
+  mediapipe::CalculatorGraphConfig config;
+  config.ParseFromArray(data.bytes, data.length);
+
+  // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
+  MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
+  [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
+  [newGraph addFrameOutputStream:kMultiFaceGeometryStream outputPacketType:MPPPacketTypeRaw];
+  return newGraph;
+}
+
+#pragma mark - UIViewController methods
+
+- (void)viewDidLoad {
+  [super viewDidLoad];
+
+  _effectSwitchingHintLabel.hidden = YES;
+  _tapGestureRecognizer = [[UITapGestureRecognizer alloc] initWithTarget:self
+                                                                  action:@selector(handleTap)];
+  [self.view addGestureRecognizer:_tapGestureRecognizer];
+
+  // By default, render the glasses effect.
+  _isFacepaintEffectSelected = NO;
+
+  _renderer = [[MPPLayerRenderer alloc] init];
+  _renderer.layer.frame = _liveView.layer.bounds;
+  [_liveView.layer insertSublayer:_renderer.layer atIndex:0];
+  _renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop;
+  _renderer.mirrored = NO;
+
+  dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class(
+      DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0);
+  _videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute);
+
+  _cameraSource = [[MPPCameraInputSource alloc] init];
+  [_cameraSource setDelegate:self queue:_videoQueue];
+  _cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
+  _cameraSource.cameraPosition = AVCaptureDevicePositionFront;
+  // The frame's native format is rotated with respect to the portrait orientation.
+  _cameraSource.orientation = AVCaptureVideoOrientationPortrait;
+  _cameraSource.videoMirrored = YES;
+
+  self.graph = [[self class] loadGraphFromResource:kGraphName];
+  self.graph.delegate = self;
+  // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
+  self.graph.maxFramesInFlight = 2;
+}
+
+// In this application, there is only one ViewController which has no navigation to other view
+// controllers, and there is only one View with live display showing the result of running the
+// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph
+// setup/teardown and camera start/stop logic should be updated appropriately in response to the
+// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times
+// depending on the application navigation flow in that case.
+- (void)viewWillAppear:(BOOL)animated {
+  [super viewWillAppear:animated];
+
+  [_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
+    if (granted) {
+      [self startGraphAndCamera];
+      dispatch_async(dispatch_get_main_queue(), ^{
+        _noCameraLabel.hidden = YES;
+      });
+    }
+  }];
+}
+
+- (void)startGraphAndCamera {
+  // Start running self.graph.
+  NSError* error;
+  if (![self.graph startWithError:&error]) {
+    NSLog(@"Failed to start graph: %@", error);
+  }
+
+  // Start fetching frames from the camera.
+  dispatch_async(_videoQueue, ^{
+    [_cameraSource start];
+  });
+}
+
+#pragma mark - UITapGestureRecognizer methods
+
+// We use the tap gesture recognizer to switch between face effects. This allows users to try
+// multiple pre-bundled face effects without a need to recompile the app.
+- (void)handleTap {
+  dispatch_async(_videoQueue, ^{
+    _isFacepaintEffectSelected = !_isFacepaintEffectSelected;
+  });
+}
+
+#pragma mark - MPPGraphDelegate methods
+
+// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+    didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
+              fromStream:(const std::string&)streamName {
+  if (streamName == kOutputStream) {
+    // Display the captured image on the screen.
+    CVPixelBufferRetain(pixelBuffer);
+    dispatch_async(dispatch_get_main_queue(), ^{
+      _effectSwitchingHintLabel.hidden = NO;
+      [_renderer renderPixelBuffer:pixelBuffer];
+      CVPixelBufferRelease(pixelBuffer);
+    });
+  }
+}
+
+// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+//
+// This callback demonstrates how the output face geometry packet can be obtained and used in an
+// iOS app. As an example, the Z-translation component of the face pose transform matrix is logged
+// for each face being equal to the approximate distance away from the camera in centimeters.
+- (void)mediapipeGraph:(MPPGraph*)graph
+     didOutputPacket:(const ::mediapipe::Packet&)packet
+          fromStream:(const std::string&)streamName {
+  if (streamName == kMultiFaceGeometryStream) {
+    if (packet.IsEmpty()) {
+      NSLog(@"[TS:%lld] No face geometry", packet.Timestamp().Value());
+      return;
+    }
+
+    const auto& multiFaceGeometry =
+        packet.Get<std::vector<::mediapipe::face_geometry::FaceGeometry>>();
+    NSLog(@"[TS:%lld] Number of face instances with geometry: %lu ", packet.Timestamp().Value(),
+          multiFaceGeometry.size());
+    for (int faceIndex = 0; faceIndex < multiFaceGeometry.size(); ++faceIndex) {
+      const auto& faceGeometry = multiFaceGeometry[faceIndex];
+      NSLog(@"\tApprox. distance away from camera for face[%d]: %.6f cm", faceIndex,
+            -faceGeometry.pose_transform_matrix().packed_data(kMatrixTranslationZIndex));
+    }
+  }
+}
+
+#pragma mark - MPPInputSourceDelegate methods
+
+// Must be invoked on _videoQueue.
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
+                timestamp:(CMTime)timestamp
+               fromSource:(MPPInputSource*)source {
+  if (source != _cameraSource) {
+    NSLog(@"Unknown source: %@", source);
+    return;
+  }
+
+  mediapipe::Timestamp graphTimestamp(static_cast<mediapipe::TimestampBaseType>(
+      mediapipe::Timestamp::kTimestampUnitsPerSecond * CMTimeGetSeconds(timestamp)));
+
+  mediapipe::Packet isFacepaintEffectSelectedPacket =
+      mediapipe::MakePacket<bool>(_isFacepaintEffectSelected).At(graphTimestamp);
+
+  [self.graph sendPixelBuffer:imageBuffer
+                   intoStream:kInputStream
+                   packetType:MPPPacketTypePixelBuffer
+                    timestamp:graphTimestamp];
+
+  // Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
+  // packet to indicate which effect should be rendered on this frame.
+  [self.graph movePacket:std::move(isFacepaintEffectSelectedPacket)
+              intoStream:kIsFacepaintEffectSelectedInputStream
+                   error:nil];
+}
+
+@end
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>NSCameraUsageDescription</key>
+  <string>This app uses the camera to demonstrate live video processing.</string>
+  <key>CFBundleDevelopmentRegion</key>
+  <string>en</string>
+  <key>CFBundleExecutable</key>
+  <string>$(EXECUTABLE_NAME)</string>
+  <key>CFBundleIdentifier</key>
+  <string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+  <key>CFBundleInfoDictionaryVersion</key>
+  <string>6.0</string>
+  <key>CFBundleName</key>
+  <string>$(PRODUCT_NAME)</string>
+  <key>CFBundlePackageType</key>
+  <string>APPL</string>
+  <key>CFBundleShortVersionString</key>
+  <string>1.0</string>
+  <key>CFBundleVersion</key>
+  <string>1</string>
+  <key>LSRequiresIPhoneOS</key>
+  <true/>
+  <key>UILaunchStoryboardName</key>
+  <string>LaunchScreen</string>
+  <key>UIMainStoryboardFile</key>
+  <string>Main</string>
+  <key>UIRequiredDeviceCapabilities</key>
+  <array>
+    <string>armv7</string>
+  </array>
+  <key>UISupportedInterfaceOrientations</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+  <key>UISupportedInterfaceOrientations~ipad</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+</dict>
+</plist>
@@ -0,0 +1,22 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char* argv[]) {
+  @autoreleasepool {
+    return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
+  }
+}
@@ -150,7 +150,7 @@ class UpperBodyPoseTracker:
      success, input_frame = cap.read()
      if not success:
        break
-      input_frame = cv2.cvtColor(input_frame, cv2.COLOR_BGR2RGB)
+      input_frame = cv2.cvtColor(cv2.flip(input_frame, 1), cv2.COLOR_BGR2RGB)
      input_frame.flags.writeable = False
      _, output_frame = self._run_graph(input_frame)
      cv2.imshow('MediaPipe upper body pose tracker',