Project import generated by Copybara.

GitOrigin-RevId: 72ff4ae24943c2ccf9905bc9e516042b0aa3dd86
2020-04-13 20:15:59 -04:00
parent 4c68eb4a70
commit 16e5d7242d
112 changed files with 4762 additions and 217 deletions
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.apps.facemeshgpu">
+
+  <uses-sdk
+      android:minSdkVersion="21"
+      android:targetSdkVersion="27" />
+
+  <!-- For using the camera -->
+  <uses-permission android:name="android.permission.CAMERA" />
+  <uses-feature android:name="android.hardware.camera" />
+  <uses-feature android:name="android.hardware.camera.autofocus" />
+  <!-- For MediaPipe -->
+  <uses-feature android:glEsVersion="0x00020000" android:required="true" />
+
+
+  <application
+      android:allowBackup="true"
+      android:label="@string/app_name"
+      android:supportsRtl="true"
+      android:theme="@style/AppTheme">
+      <activity
+          android:name=".MainActivity"
+          android:exported="true"
+          android:screenOrientation="portrait">
+          <intent-filter>
+              <action android:name="android.intent.action.MAIN" />
+              <category android:name="android.intent.category.LAUNCHER" />
+          </intent-filter>
+      </activity>
+  </application>
+
+</manifest>
@@ -0,0 +1,82 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "libmediapipe_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/graphs/face_mesh:mobile_calculators",
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+    ],
+)
+
+cc_library(
+    name = "mediapipe_jni_lib",
+    srcs = [":libmediapipe_jni.so"],
+    alwayslink = 1,
+)
+
+# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be
+# easily incorporated into the app via, for example,
+# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb".
+genrule(
+    name = "binary_graph",
+    srcs = ["//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu_binary_graph"],
+    outs = ["facemeshgpu.binarypb"],
+    cmd = "cp $< $@",
+)
+
+android_library(
+    name = "mediapipe_lib",
+    srcs = glob(["*.java"]),
+    assets = [
+        ":binary_graph",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+    ],
+    assets_dir = "",
+    manifest = "AndroidManifest.xml",
+    resource_files = glob(["res/**"]),
+    deps = [
+        ":mediapipe_jni_lib",
+        "//mediapipe/framework/formats:landmark_java_proto_lite",
+        "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
+        "//mediapipe/java/com/google/mediapipe/components:android_components",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/java/com/google/mediapipe/glutil",
+        "//third_party:androidx_appcompat",
+        "//third_party:androidx_constraint_layout",
+        "//third_party:androidx_legacy_support_v4",
+        "//third_party:androidx_recyclerview",
+        "//third_party:opencv",
+        "@maven//:androidx_concurrent_concurrent_futures",
+        "@maven//:androidx_lifecycle_lifecycle_common",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
+android_binary(
+    name = "facemeshgpu",
+    manifest = "AndroidManifest.xml",
+    manifest_values = {"applicationId": "com.google.mediapipe.apps.facemeshgpu"},
+    multidex = "native",
+    deps = [
+        ":mediapipe_lib",
+    ],
+)
@@ -0,0 +1,232 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.apps.facemeshgpu;
+
+import android.graphics.SurfaceTexture;
+import android.os.Bundle;
+import androidx.appcompat.app.AppCompatActivity;
+import android.util.Log;
+import android.util.Size;
+import android.view.SurfaceHolder;
+import android.view.SurfaceView;
+import android.view.View;
+import android.view.ViewGroup;
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
+import com.google.mediapipe.components.CameraHelper;
+import com.google.mediapipe.components.CameraXPreviewHelper;
+import com.google.mediapipe.components.ExternalTextureConverter;
+import com.google.mediapipe.components.FrameProcessor;
+import com.google.mediapipe.components.PermissionHelper;
+import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.framework.AndroidPacketCreator;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.framework.PacketGetter;
+import com.google.mediapipe.glutil.EglManager;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Main activity of MediaPipe example apps. */
+public class MainActivity extends AppCompatActivity {
+  private static final String TAG = "MainActivity";
+
+  private static final String BINARY_GRAPH_NAME = "facemeshgpu.binarypb";
+  private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
+  private static final String INPUT_NUM_FACES_SIDE_PACKET_NAME = "num_faces";
+  private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final String OUTPUT_LANDMARKS_STREAM_NAME = "multi_face_landmarks";
+  private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.FRONT;
+
+  // Max number of faces to detect/process.
+  private static final int NUM_FACES = 1;
+
+  // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
+  // processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
+  // This is needed because OpenGL represents images assuming the image origin is at the bottom-left
+  // corner, whereas MediaPipe in general assumes the image origin is at top-left.
+  private static final boolean FLIP_FRAMES_VERTICALLY = true;
+
+  static {
+    // Load all native libraries needed by the app.
+    System.loadLibrary("mediapipe_jni");
+    System.loadLibrary("opencv_java3");
+  }
+
+  // {@link SurfaceTexture} where the camera-preview frames can be accessed.
+  private SurfaceTexture previewFrameTexture;
+  // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
+  private SurfaceView previewDisplayView;
+
+  // Creates and manages an {@link EGLContext}.
+  private EglManager eglManager;
+  // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
+  // frames onto a {@link Surface}.
+  private FrameProcessor processor;
+  // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
+  // consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
+  private ExternalTextureConverter converter;
+
+  // Handles camera access via the {@link CameraX} Jetpack support library.
+  private CameraXPreviewHelper cameraHelper;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_main);
+
+    previewDisplayView = new SurfaceView(this);
+    setupPreviewDisplayView();
+
+    // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
+    // binary graphs.
+    AndroidAssetUtil.initializeNativeAssetManager(this);
+
+    eglManager = new EglManager(null);
+    processor =
+        new FrameProcessor(
+            this,
+            eglManager.getNativeContext(),
+            BINARY_GRAPH_NAME,
+            INPUT_VIDEO_STREAM_NAME,
+            OUTPUT_VIDEO_STREAM_NAME);
+    processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
+
+    AndroidPacketCreator packetCreator = processor.getPacketCreator();
+    Map<String, Packet> inputSidePackets = new HashMap<>();
+    inputSidePackets.put(INPUT_NUM_FACES_SIDE_PACKET_NAME, packetCreator.createInt32(NUM_FACES));
+    processor.setInputSidePackets(inputSidePackets);
+
+    processor.addPacketCallback(
+        OUTPUT_LANDMARKS_STREAM_NAME,
+        (packet) -> {
+          Log.d(TAG, "Received multi face landmarks packet.");
+          List<NormalizedLandmarkList> multiFaceLandmarks =
+              PacketGetter.getProtoVector(packet, NormalizedLandmarkList.parser());
+          Log.d(
+              TAG,
+              "[TS:"
+                  + packet.getTimestamp()
+                  + "] "
+                  + getMultiFaceLandmarksDebugString(multiFaceLandmarks));
+        });
+
+    PermissionHelper.checkAndRequestCameraPermissions(this);
+  }
+
+  @Override
+  protected void onResume() {
+    super.onResume();
+    converter = new ExternalTextureConverter(eglManager.getContext());
+    converter.setFlipY(FLIP_FRAMES_VERTICALLY);
+    converter.setConsumer(processor);
+    if (PermissionHelper.cameraPermissionsGranted(this)) {
+      startCamera();
+    }
+  }
+
+  @Override
+  protected void onPause() {
+    super.onPause();
+    converter.close();
+  }
+
+  @Override
+  public void onRequestPermissionsResult(
+      int requestCode, String[] permissions, int[] grantResults) {
+    super.onRequestPermissionsResult(requestCode, permissions, grantResults);
+    PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
+  }
+
+  private void setupPreviewDisplayView() {
+    previewDisplayView.setVisibility(View.GONE);
+    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
+    viewGroup.addView(previewDisplayView);
+
+    previewDisplayView
+        .getHolder()
+        .addCallback(
+            new SurfaceHolder.Callback() {
+              @Override
+              public void surfaceCreated(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
+              }
+
+              @Override
+              public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+                // (Re-)Compute the ideal size of the camera-preview display (the area that the
+                // camera-preview frames get rendered onto, potentially with scaling and rotation)
+                // based on the size of the SurfaceView that contains the display.
+                Size viewSize = new Size(width, height);
+                Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
+
+                // Connect the converter to the camera-preview frames as its input (via
+                // previewFrameTexture), and configure the output width and height as the computed
+                // display size.
+                converter.setSurfaceTextureAndAttachToGLContext(
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
+              }
+
+              @Override
+              public void surfaceDestroyed(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(null);
+              }
+            });
+  }
+
+  private void startCamera() {
+    cameraHelper = new CameraXPreviewHelper();
+    cameraHelper.setOnCameraStartedListener(
+        surfaceTexture -> {
+          previewFrameTexture = surfaceTexture;
+          // Make the display view visible to start showing the preview. This triggers the
+          // SurfaceHolder.Callback added to (the holder of) previewDisplayView.
+          previewDisplayView.setVisibility(View.VISIBLE);
+        });
+    cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
+  }
+
+  private static String getMultiFaceLandmarksDebugString(
+      List<NormalizedLandmarkList> multiFaceLandmarks) {
+    if (multiFaceLandmarks.isEmpty()) {
+      return "No face landmarks";
+    }
+    String multiFaceLandmarksStr = "Number of faces detected: " + multiFaceLandmarks.size() + "\n";
+    int faceIndex = 0;
+    for (NormalizedLandmarkList landmarks : multiFaceLandmarks) {
+      multiFaceLandmarksStr +=
+          "\t#Face landmarks for face[" + faceIndex + "]: " + landmarks.getLandmarkCount() + "\n";
+      int landmarkIndex = 0;
+      for (NormalizedLandmark landmark : landmarks.getLandmarkList()) {
+        multiFaceLandmarksStr +=
+            "\t\tLandmark ["
+                + landmarkIndex
+                + "]: ("
+                + landmark.getX()
+                + ", "
+                + landmark.getY()
+                + ", "
+                + landmark.getZ()
+                + ")\n";
+        ++landmarkIndex;
+      }
+      ++faceIndex;
+    }
+    return multiFaceLandmarksStr;
+  }
+}
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <FrameLayout
+        android:id="@+id/preview_display_layout"
+        android:layout_width="fill_parent"
+        android:layout_height="fill_parent"
+        android:layout_weight="1">
+        <TextView
+            android:id="@+id/no_camera_access_view"
+            android:layout_height="fill_parent"
+            android:layout_width="fill_parent"
+            android:gravity="center"
+            android:text="@string/no_camera_access" />
+    </FrameLayout>
+</androidx.constraintlayout.widget.ConstraintLayout>
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <color name="colorPrimary">#008577</color>
+    <color name="colorPrimaryDark">#00574B</color>
+    <color name="colorAccent">#D81B60</color>
+</resources>
@@ -0,0 +1,4 @@
+<resources>
+    <string name="app_name" translatable="false">Face Mesh GPU</string>
+    <string name="no_camera_access" translatable="false">Please grant camera permissions.</string>
+</resources>
@@ -0,0 +1,11 @@
+<resources>
+
+    <!-- Base application theme. -->
+    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+        <!-- Customize your theme here. -->
+        <item name="colorPrimary">@color/colorPrimary</item>
+        <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
+        <item name="colorAccent">@color/colorAccent</item>
+    </style>
+
+</resources>
@@ -150,4 +150,34 @@ message ConversionOptions {
  optional int32 target_height = 2;
 }

-// TODO: Move other autoflip messages into this area.
+// Self-contained message that provides all needed information to render
+// autoflip with an external renderer.  One of these messages is required for
+// each frame of the video.
+message ExternalRenderFrame {
+  // Rectangle using opencv standard.
+  message Rect {
+    optional float x = 1;
+    optional float y = 2;
+    optional float width = 3;
+    optional float height = 4;
+  }
+  // RGB color [0...255]
+  message Color {
+    optional int32 r = 1;
+    optional int32 g = 2;
+    optional int32 b = 3;
+  }
+  // Rect that must be cropped out of the input frame.  It is in the
+  // original dimensions of the input video.  The first step to render this
+  // frame is to crop this rect from the input frame.
+  optional Rect crop_from_location = 1;
+  // The placement location where the above rect is placed on the output frame.
+  // This will always have the same aspect ratio as the above rect but scaling
+  // may be required.
+  optional Rect render_to_location = 2;
+  // If render_to_location is smaller than the output dimensions of the frame,
+  // fill the rest of the frame with this color.
+  optional Color padding_color = 3;
+  // Timestamp in microseconds of this frame.
+  optional uint64 timestamp_us = 4;
+}
@@ -44,11 +44,19 @@ constexpr char kInputExternalSettings[] = "EXTERNAL_SETTINGS";
 // TargetSizeType::MAXIMIZE_TARGET_DIMENSION
 constexpr char kAspectRatio[] = "EXTERNAL_ASPECT_RATIO";

+// Output the cropped frames, as well as visualization of crop regions and focus
+// points. Note that, KEY_FRAME_CROP_REGION_VIZ_FRAMES and
+// SALIENT_POINT_FRAME_VIZ_FRAMES can only be enabled when CROPPED_FRAMES is
+// enabled.
 constexpr char kOutputCroppedFrames[] = "CROPPED_FRAMES";
 constexpr char kOutputKeyFrameCropViz[] = "KEY_FRAME_CROP_REGION_VIZ_FRAMES";
 constexpr char kOutputFocusPointFrameViz[] = "SALIENT_POINT_FRAME_VIZ_FRAMES";
 constexpr char kOutputSummary[] = "CROPPING_SUMMARY";

+// External rendering outputs
+constexpr char kExternalRenderingPerFrame[] = "EXTERNAL_RENDERING_PER_FRAME";
+constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
+
 ::mediapipe::Status SceneCroppingCalculator::GetContract(
    ::mediapipe::CalculatorContract* cc) {
  if (cc->InputSidePackets().HasTag(kInputExternalSettings)) {
@@ -67,16 +75,36 @@ constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
  }
  cc->Inputs().Tag(kInputShotBoundaries).Set<bool>();

-  cc->Outputs().Tag(kOutputCroppedFrames).Set<ImageFrame>();
+  if (cc->Outputs().HasTag(kOutputCroppedFrames)) {
+    cc->Outputs().Tag(kOutputCroppedFrames).Set<ImageFrame>();
+  }
  if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
+    RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames))
+        << "KEY_FRAME_CROP_REGION_VIZ_FRAMES can only be used when "
+           "CROPPED_FRAMES is specified.";
    cc->Outputs().Tag(kOutputKeyFrameCropViz).Set<ImageFrame>();
  }
  if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
+    RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames))
+        << "SALIENT_POINT_FRAME_VIZ_FRAMES can only be used when "
+           "CROPPED_FRAMES is specified.";
    cc->Outputs().Tag(kOutputFocusPointFrameViz).Set<ImageFrame>();
  }
  if (cc->Outputs().HasTag(kOutputSummary)) {
    cc->Outputs().Tag(kOutputSummary).Set<VideoCroppingSummary>();
  }
+  if (cc->Outputs().HasTag(kExternalRenderingPerFrame)) {
+    cc->Outputs().Tag(kExternalRenderingPerFrame).Set<ExternalRenderFrame>();
+  }
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    cc->Outputs()
+        .Tag(kExternalRenderingFullVid)
+        .Set<std::vector<ExternalRenderFrame>>();
+  }
+  RET_CHECK(cc->Outputs().HasTag(kExternalRenderingPerFrame) ||
+            cc->Outputs().HasTag(kExternalRenderingFullVid) ||
+            cc->Outputs().HasTag(kOutputCroppedFrames))
+      << "At leaset one output stream must be specified";
  return ::mediapipe::OkStatus();
 }

@@ -104,6 +132,11 @@ constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
  if (cc->Outputs().HasTag(kOutputSummary)) {
    summary_ = absl::make_unique<VideoCroppingSummary>();
  }
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    external_render_list_ =
+        absl::make_unique<std::vector<ExternalRenderFrame>>();
+  }
+  should_perform_frame_cropping_ = cc->Outputs().HasTag(kOutputCroppedFrames);
  return ::mediapipe::OkStatus();
 }

@@ -127,6 +160,28 @@ namespace {
  *aspect_ratio = width_ratio / height_ratio;
  return ::mediapipe::OkStatus();
 }
+void ConstructExternalRenderMessage(
+    const cv::Rect& crop_from_location, const cv::Rect& render_to_location,
+    const cv::Scalar& padding_color, const uint64 timestamp_us,
+    ExternalRenderFrame* external_render_message) {
+  auto crop_from_message =
+      external_render_message->mutable_crop_from_location();
+  crop_from_message->set_x(crop_from_location.x);
+  crop_from_message->set_y(crop_from_location.y);
+  crop_from_message->set_width(crop_from_location.width);
+  crop_from_message->set_height(crop_from_location.height);
+  auto render_to_message =
+      external_render_message->mutable_render_to_location();
+  render_to_message->set_x(render_to_location.x);
+  render_to_message->set_y(render_to_location.y);
+  render_to_message->set_width(render_to_location.width);
+  render_to_message->set_height(render_to_location.height);
+  auto padding_color_message = external_render_message->mutable_padding_color();
+  padding_color_message->set_r(padding_color[0]);
+  padding_color_message->set_g(padding_color[1]);
+  padding_color_message->set_b(padding_color[2]);
+  external_render_message->set_timestamp_us(timestamp_us);
+}
 }  // namespace

 ::mediapipe::Status SceneCroppingCalculator::Process(
@@ -230,8 +285,9 @@ namespace {
    is_end_of_scene = cc->Inputs().Tag(kInputShotBoundaries).Get<bool>();
  }
  const bool force_buffer_flush =
-      scene_frames_.size() >= options_.max_scene_size();
-  if (!scene_frames_.empty() && (is_end_of_scene || force_buffer_flush)) {
+      scene_frame_timestamps_.size() >= options_.max_scene_size();
+  if (!scene_frame_timestamps_.empty() &&
+      (is_end_of_scene || force_buffer_flush)) {
    MP_RETURN_IF_ERROR(ProcessScene(is_end_of_scene, cc));
  }

@@ -240,11 +296,14 @@ namespace {
    LOG_EVERY_N(ERROR, 10)
        << "------------------------ (Breathing) Time(s): "
        << cc->Inputs().Tag(kInputVideoFrames).Value().Timestamp().Seconds();
-    const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
-    const cv::Mat frame_mat = formats::MatView(&frame);
-    cv::Mat copy_mat;
-    frame_mat.copyTo(copy_mat);
-    scene_frames_.push_back(copy_mat);
+    // Only buffer frames if |should_perform_frame_cropping_| is true.
+    if (should_perform_frame_cropping_) {
+      const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
+      const cv::Mat frame_mat = formats::MatView(&frame);
+      cv::Mat copy_mat;
+      frame_mat.copyTo(copy_mat);
+      scene_frames_or_empty_.push_back(copy_mat);
+    }
    scene_frame_timestamps_.push_back(cc->InputTimestamp().Value());
    is_key_frames_.push_back(
        !cc->Inputs().Tag(kInputDetections).Value().IsEmpty());
@@ -274,7 +333,7 @@ namespace {

 ::mediapipe::Status SceneCroppingCalculator::Close(
    ::mediapipe::CalculatorContext* cc) {
-  if (!scene_frames_.empty()) {
+  if (!scene_frame_timestamps_.empty()) {
    MP_RETURN_IF_ERROR(ProcessScene(/* is_end_of_scene = */ true, cc));
  }
  if (cc->Outputs().HasTag(kOutputSummary)) {
@@ -282,16 +341,25 @@ namespace {
        .Tag(kOutputSummary)
        .Add(summary_.release(), Timestamp::PostStream());
  }
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    cc->Outputs()
+        .Tag(kExternalRenderingFullVid)
+        .Add(external_render_list_.release(), Timestamp::PostStream());
+  }
  return ::mediapipe::OkStatus();
 }

-::mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders() {
-  int top_border_size = 0, bottom_border_size = 0;
+// TODO: split this function into two, one for calculating the border
+// sizes, the other for the actual removal of borders from the frames.
+::mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders(
+    int* top_border_size, int* bottom_border_size) {
+  *top_border_size = 0;
+  *bottom_border_size = 0;
  MP_RETURN_IF_ERROR(ComputeSceneStaticBordersSize(
-      static_features_, &top_border_size, &bottom_border_size));
+      static_features_, top_border_size, bottom_border_size));
  const double scale = static_cast<double>(frame_height_) / key_frame_height_;
-  top_border_distance_ = std::round(scale * top_border_size);
-  const int bottom_border_distance = std::round(scale * bottom_border_size);
+  top_border_distance_ = std::round(scale * *top_border_size);
+  const int bottom_border_distance = std::round(scale * *bottom_border_size);
  effective_frame_height_ =
      frame_height_ - top_border_distance_ - bottom_border_distance;

@@ -301,10 +369,10 @@ namespace {
    // Remove borders from frames.
    cv::Rect roi(0, top_border_distance_, frame_width_,
                 effective_frame_height_);
-    for (int i = 0; i < scene_frames_.size(); ++i) {
+    for (int i = 0; i < scene_frames_or_empty_.size(); ++i) {
      cv::Mat tmp;
-      scene_frames_[i](roi).copyTo(tmp);
-      scene_frames_[i] = tmp;
+      scene_frames_or_empty_[i](roi).copyTo(tmp);
+      scene_frames_or_empty_[i] = tmp;
    }
    // Adjust detection bounding boxes.
    for (int i = 0; i < key_frame_infos_.size(); ++i) {
@@ -373,7 +441,9 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
  FilterKeyFrameInfo();

  // Removes any static borders.
-  MP_RETURN_IF_ERROR(RemoveStaticBorders());
+  int top_static_border_size, bottom_static_border_size;
+  MP_RETURN_IF_ERROR(
+      RemoveStaticBorders(&top_static_border_size, &bottom_static_border_size));

  // Decides if solid background color padding is possible and sets up color
  // interpolation functions in CIELAB. Uses linear interpolation by default.
@@ -409,21 +479,32 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {

  // Crops scene frames.
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+
+  auto* cropped_frames_ptr =
+      should_perform_frame_cropping_ ? &cropped_frames : nullptr;
+
  MP_RETURN_IF_ERROR(scene_cropper_->CropFrames(
-      scene_summary, scene_frames_, focus_point_frames,
-      prior_focus_point_frames_, &cropped_frames));
+      scene_summary, scene_frame_timestamps_.size(), scene_frames_or_empty_,
+      focus_point_frames, prior_focus_point_frames_, top_static_border_size,
+      bottom_static_border_size, &crop_from_locations, cropped_frames_ptr));

  // Formats and outputs cropped frames.
  bool apply_padding = false;
  float vertical_fill_precent;
-  MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
-      cropped_frames, &apply_padding, &vertical_fill_precent, cc));
-
+  std::vector<cv::Rect> render_to_locations;
+  cv::Scalar padding_color;
+  if (should_perform_frame_cropping_) {
+    MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
+        cropped_frames, &render_to_locations, &apply_padding, &padding_color,
+        &vertical_fill_precent, cc));
+  }
  // Caches prior FocusPointFrames if this was not the end of a scene.
  prior_focus_point_frames_.clear();
  if (!is_end_of_scene) {
-    const int start = std::max(0, static_cast<int>(scene_frames_.size()) -
-                                      options_.prior_frame_buffer_size());
+    const int start =
+        std::max(0, static_cast<int>(scene_frame_timestamps_.size()) -
+                        options_.prior_frame_buffer_size());
    for (int i = start; i < num_key_frames; ++i) {
      prior_focus_point_frames_.push_back(focus_point_frames[i]);
    }
@@ -449,8 +530,31 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
    scene_summary->set_is_padded(apply_padding);
  }

+  if (cc->Outputs().HasTag(kExternalRenderingPerFrame)) {
+    for (int i = 0; i < scene_frame_timestamps_.size(); i++) {
+      auto external_render_message = absl::make_unique<ExternalRenderFrame>();
+      ConstructExternalRenderMessage(
+          crop_from_locations[i], render_to_locations[i], padding_color,
+          scene_frame_timestamps_[i], external_render_message.get());
+      cc->Outputs()
+          .Tag(kExternalRenderingPerFrame)
+          .Add(external_render_message.release(),
+               Timestamp(scene_frame_timestamps_[i]));
+    }
+  }
+
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    for (int i = 0; i < scene_frame_timestamps_.size(); i++) {
+      ExternalRenderFrame render_frame;
+      ConstructExternalRenderMessage(crop_from_locations[i],
+                                     render_to_locations[i], padding_color,
+                                     scene_frame_timestamps_[i], &render_frame);
+      external_render_list_->push_back(render_frame);
+    }
+  }
+
  key_frame_infos_.clear();
-  scene_frames_.clear();
+  scene_frames_or_empty_.clear();
  scene_frame_timestamps_.clear();
  is_key_frames_.clear();
  static_features_.clear();
@@ -459,8 +563,10 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
 }

 ::mediapipe::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
-    const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
-    float* vertical_fill_precent, CalculatorContext* cc) {
+    const std::vector<cv::Mat>& cropped_frames,
+    std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
+    cv::Scalar* padding_color, float* vertical_fill_precent,
+    CalculatorContext* cc) {
  RET_CHECK(apply_padding) << "Has padding boolean is null.";
  if (cropped_frames.empty()) {
    return ::mediapipe::OkStatus();
@@ -493,10 +599,22 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
            << " target height = " << target_height_;
  }

+  // Compute the "render to" location.  This is where the rect taken from the
+  // input video gets pasted on the output frame.  For use with external
+  // rendering solutions.
+  const int num_frames = cropped_frames.size();
+  for (int i = 0; i < num_frames; i++) {
+    if (*apply_padding) {
+      render_to_locations->push_back(padder_->ComputeOutputLocation());
+    } else {
+      render_to_locations->push_back(
+          cv::Rect(0, 0, target_width_, target_height_));
+    }
+  }
+
  // Resizes cropped frames, pads frames, and output frames.
  cv::Scalar* background_color = nullptr;
  cv::Scalar interpolated_color;
-  const int num_frames = cropped_frames.size();
  for (int i = 0; i < num_frames; ++i) {
    const int64 time_ms = scene_frame_timestamps_[i];
    const Timestamp timestamp(time_ms);
@@ -561,9 +679,9 @@ mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
  if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
    std::vector<std::unique_ptr<ImageFrame>> viz_frames;
    MP_RETURN_IF_ERROR(DrawDetectionsAndCropRegions(
-        scene_frames_, is_key_frames_, key_frame_infos_, key_frame_crop_results,
-        frame_format_, &viz_frames));
-    for (int i = 0; i < scene_frames_.size(); ++i) {
+        scene_frames_or_empty_, is_key_frames_, key_frame_infos_,
+        key_frame_crop_results, frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_or_empty_.size(); ++i) {
      cc->Outputs()
          .Tag(kOutputKeyFrameCropViz)
          .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
@@ -572,9 +690,10 @@ mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
  if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
    std::vector<std::unique_ptr<ImageFrame>> viz_frames;
    MP_RETURN_IF_ERROR(DrawFocusPointAndCropWindow(
-        scene_frames_, focus_point_frames, options_.viz_overlay_opacity(),
-        crop_window_width, crop_window_height, frame_format_, &viz_frames));
-    for (int i = 0; i < scene_frames_.size(); ++i) {
+        scene_frames_or_empty_, focus_point_frames,
+        options_.viz_overlay_opacity(), crop_window_width, crop_window_height,
+        frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_or_empty_.size(); ++i) {
      cc->Outputs()
          .Tag(kOutputFocusPointFrameViz)
          .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
@@ -79,8 +79,10 @@ namespace autoflip {
 //     Indicators for shot boundaries (output of shot boundary detection).
 // - optional tag KEY_FRAMES (type ImageFrame):
 //     Key frames on which features are detected. This is only used to set the
-//     detection features frame size, and when it is omitted, the features frame
-//     size is assumed to be the original scene frame size.
+//     detection features frame size.  Alternatively, set
+//     video_feature_width/video_features_height within the options proto to
+//     define this value.  When neither is set, the features frame size is
+//     assumed to be the original scene frame size.
 //
 // Output streams:
 // - required tag CROPPED_FRAMES (type ImageFrame):
@@ -95,6 +97,12 @@ namespace autoflip {
 // - optional tag CROPPING_SUMMARY (type VideoCroppingSummary):
 //     Debug summary information for the video. Only generates one packet when
 //     calculator closes.
+// - optional tag EXTERNAL_RENDERING_PER_FRAME (type ExternalRenderFrame)
+//     Provides a per-frame message that can be used to render autoflip using an
+//     external renderer.
+// - optional tag EXTERNAL_RENDERING_FULL_VID (type Vector<ExternalRenderFrame>)
+//     Provides an end-stream message that can be used to render autoflip using
+//     an external renderer.
 //
 // Example config:
 // node {
@@ -134,8 +142,11 @@ class SceneCroppingCalculator : public CalculatorBase {
  ::mediapipe::Status Close(::mediapipe::CalculatorContext* cc) override;

 private:
-  // Removes any static borders from the scene frames before cropping.
-  ::mediapipe::Status RemoveStaticBorders();
+  // Removes any static borders from the scene frames before cropping. The
+  // arguments |top_border_size| and |bottom_border_size| report the size of the
+  // removed borders.
+  ::mediapipe::Status RemoveStaticBorders(int* top_border_size,
+                                          int* bottom_border_size);

  // Initializes a FrameCropRegionComputer given input and target frame sizes.
  ::mediapipe::Status InitializeFrameCropRegionComputer();
@@ -158,8 +169,10 @@ class SceneCroppingCalculator : public CalculatorBase {
  // solid background from static features if possible, otherwise uses blurred
  // background. Sets apply_padding to true if the scene is padded.
  ::mediapipe::Status FormatAndOutputCroppedFrames(
-      const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
-      float* vertical_fill_precent, CalculatorContext* cc);
+      const std::vector<cv::Mat>& cropped_frames,
+      std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
+      cv::Scalar* padding_color, float* vertical_fill_precent,
+      CalculatorContext* cc);

  // Draws and outputs visualization frames if those streams are present.
  ::mediapipe::Status OutputVizFrames(
@@ -193,7 +206,11 @@ class SceneCroppingCalculator : public CalculatorBase {

  // Buffered frames, timestamps, and indicators for key frames in the current
  // scene (size = number of input video frames).
-  std::vector<cv::Mat> scene_frames_;
+  // Note: scene_frames_or_empty_ may be empty if the actual cropping operation
+  // of frames is turned off, e.g. when |should_perform_frame_cropping_| is
+  // false, so rely on scene_frame_timestamps_.size() to query the number of
+  // accumulated timestamps rather than scene_frames_or_empty_.size().
+  std::vector<cv::Mat> scene_frames_or_empty_;
  std::vector<int64> scene_frame_timestamps_;
  std::vector<bool> is_key_frames_;

@@ -242,6 +259,17 @@ class SceneCroppingCalculator : public CalculatorBase {

  // Optional diagnostic summary output emitted in Close().
  std::unique_ptr<VideoCroppingSummary> summary_ = nullptr;
+
+  // Optional list of external rendering messages for each processed frame.
+  std::unique_ptr<std::vector<ExternalRenderFrame>> external_render_list_;
+
+  // Determines whether to perform real cropping on input frames. This flag is
+  // useful when the user only needs to compute cropping windows, in which case
+  // setting this flag to false can avoid buffering as well as cropping frames.
+  // This can significantly reduce memory usage and speed up processing. Some
+  // debugging visualization inevitably will be disabled because of this flag
+  // too.
+  bool should_perform_frame_cropping_ = false;
 };
 }  // namespace autoflip
 }  // namespace mediapipe
@@ -68,6 +68,22 @@ constexpr char kNoKeyFrameConfig[] = R"(
    }
  })";

+constexpr char kDebugConfigNoCroppedFrame[] = R"(
+  calculator: "SceneCroppingCalculator"
+  input_stream: "VIDEO_FRAMES:camera_frames_org"
+  input_stream: "KEY_FRAMES:down_sampled_frames"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:border_features"
+  input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
+  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      target_width: $0
+      target_height: $1
+    }
+  })";
+
 constexpr char kDebugConfig[] = R"(
  calculator: "SceneCroppingCalculator"
  input_stream: "VIDEO_FRAMES:camera_frames_org"
@@ -79,6 +95,8 @@ constexpr char kDebugConfig[] = R"(
  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
  output_stream: "CROPPING_SUMMARY:cropping_summaries"
+  output_stream: "EXTERNAL_RENDERING_PER_FRAME:external_rendering_per_frame"
+  output_stream: "EXTERNAL_RENDERING_FULL_VID:external_rendering_full_vid"
  options: {
    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
      target_width: $0
@@ -257,6 +275,17 @@ TEST(SceneCroppingCalculatorTest, ChecksPriorFrameBufferSize) {
              HasSubstr("Prior frame buffer size is negative."));
 }

+TEST(SceneCroppingCalculatorTest, ChecksDebugConfigWithoutCroppedFrame) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kDebugConfigNoCroppedFrame, kTargetWidth, kTargetHeight,
+          kTargetSizeType, 0, kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  const auto status = runner->Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("can only be used when"));
+}
+
 // Checks that the calculator crops scene frames when there is no input key
 // frames stream.
 TEST(SceneCroppingCalculatorTest, HandlesNoKeyFrames) {
@@ -299,14 +328,34 @@ TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) {
  EXPECT_TRUE(outputs.HasTag("KEY_FRAME_CROP_REGION_VIZ_FRAMES"));
  EXPECT_TRUE(outputs.HasTag("SALIENT_POINT_FRAME_VIZ_FRAMES"));
  EXPECT_TRUE(outputs.HasTag("CROPPING_SUMMARY"));
+  EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_PER_FRAME"));
+  EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_FULL_VID"));
  const auto& crop_region_viz_frames_outputs =
      outputs.Tag("KEY_FRAME_CROP_REGION_VIZ_FRAMES").packets;
  const auto& salient_point_viz_frames_outputs =
      outputs.Tag("SALIENT_POINT_FRAME_VIZ_FRAMES").packets;
  const auto& summary_output = outputs.Tag("CROPPING_SUMMARY").packets;
+  const auto& ext_render_per_frame =
+      outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
+  const auto& ext_render_full_vid =
+      outputs.Tag("EXTERNAL_RENDERING_FULL_VID").packets;
  EXPECT_EQ(crop_region_viz_frames_outputs.size(), num_frames);
  EXPECT_EQ(salient_point_viz_frames_outputs.size(), num_frames);
  EXPECT_EQ(summary_output.size(), 1);
+  EXPECT_EQ(ext_render_per_frame.size(), num_frames);
+  EXPECT_EQ(ext_render_full_vid.size(), 1);
+  EXPECT_EQ(ext_render_per_frame[0].Get<ExternalRenderFrame>().timestamp_us(),
+            0);
+  EXPECT_EQ(ext_render_full_vid[0]
+                .Get<std::vector<ExternalRenderFrame>>()[0]
+                .timestamp_us(),
+            0);
+  EXPECT_EQ(ext_render_per_frame[1].Get<ExternalRenderFrame>().timestamp_us(),
+            20000);
+  EXPECT_EQ(ext_render_full_vid[0]
+                .Get<std::vector<ExternalRenderFrame>>()[1]
+                .timestamp_us(),
+            20000);

  for (int i = 0; i < num_frames; ++i) {
    const auto& crop_region_viz_frame =
@@ -173,5 +173,28 @@ PaddingEffectGenerator::PaddingEffectGenerator(const int input_width,
  return ::mediapipe::OkStatus();
 }

+cv::Rect PaddingEffectGenerator::ComputeOutputLocation() {
+  const int effective_input_width =
+      is_vertical_padding_ ? input_width_ : input_height_;
+  const int effective_input_height =
+      is_vertical_padding_ ? input_height_ : input_width_;
+  const int effective_output_width =
+      is_vertical_padding_ ? output_width_ : output_height_;
+  const int effective_output_height =
+      is_vertical_padding_ ? output_height_ : output_width_;
+
+  // Step 3 from "process" call above, compute foreground location.
+  const int foreground_height =
+      effective_input_height * effective_output_width / effective_input_width;
+  const int x = 0;
+  const int y = (effective_output_height - foreground_height) / 2;
+  const int width = effective_output_width;
+  const int height = foreground_height;
+
+  cv::Rect region_to_embed_foreground(x, y, width, height);
+
+  return region_to_embed_foreground;
+}
+
 }  // namespace autoflip
 }  // namespace mediapipe
@@ -55,6 +55,10 @@ class PaddingEffectGenerator {
      ImageFrame* output_frame,
      const cv::Scalar* background_color_in_rgb = nullptr);

+  // Compute the "render location" on the output frame where the "crop from"
+  // location is to be placed.  For use with external rendering soutions.
+  cv::Rect ComputeOutputLocation();
+
 private:
  double target_aspect_ratio_;
  int input_width_ = -1;
@@ -182,6 +182,16 @@ TEST(PaddingEffectGeneratorTest, ScaleToMultipleOfTwo) {
  EXPECT_EQ(result_frame.Width(), expect_width);
  EXPECT_EQ(result_frame.Height(), expect_height);
 }
+
+TEST(PaddingEffectGeneratorTest, ComputeOutputLocation) {
+  PaddingEffectGenerator generator(1920, 1080, 1.0);
+
+  auto result_rect = generator.ComputeOutputLocation();
+  EXPECT_EQ(result_rect.x, 0);
+  EXPECT_EQ(result_rect.y, 236);
+  EXPECT_EQ(result_rect.width, 1080);
+  EXPECT_EQ(result_rect.height, 607);
+}
 }  // namespace
 }  // namespace autoflip
 }  // namespace mediapipe
@@ -25,14 +25,13 @@ namespace mediapipe {
 namespace autoflip {

 ::mediapipe::Status SceneCropper::CropFrames(
-    const SceneKeyFrameCropSummary& scene_summary,
-    const std::vector<cv::Mat>& scene_frames,
+    const SceneKeyFrameCropSummary& scene_summary, const int num_scene_frames,
+    const std::vector<cv::Mat>& scene_frames_or_empty,
    const std::vector<FocusPointFrame>& focus_point_frames,
    const std::vector<FocusPointFrame>& prior_focus_point_frames,
+    int top_static_border_size, int bottom_static_border_size,
+    std::vector<cv::Rect>* crop_from_location,
    std::vector<cv::Mat>* cropped_frames) const {
-  RET_CHECK_NE(cropped_frames, nullptr) << "Output cropped frames is null.";
-
-  const int num_scene_frames = scene_frames.size();
  RET_CHECK_GT(num_scene_frames, 0) << "No scene frames.";
  RET_CHECK_EQ(focus_point_frames.size(), num_scene_frames)
      << "Wrong size of FocusPointFrames.";
@@ -69,15 +68,36 @@ namespace autoflip {
    xform = affine_opencv;
  }

+  // If no cropped_frames is passed in, return directly.
+  if (!cropped_frames) {
+    return ::mediapipe::OkStatus();
+  }
+  RET_CHECK(!scene_frames_or_empty.empty())
+      << "If |cropped_frames| != nullptr, scene_frames_or_empty must not be "
+         "empty.";
  // Prepares cropped frames.
  cropped_frames->resize(num_scene_frames);
  for (int i = 0; i < num_scene_frames; ++i) {
-    (*cropped_frames)[i] =
-        cv::Mat::zeros(crop_height, crop_width, scene_frames[i].type());
+    (*cropped_frames)[i] = cv::Mat::zeros(crop_height, crop_width,
+                                          scene_frames_or_empty[i].type());
  }

-  return AffineRetarget(cv::Size(crop_width, crop_height), scene_frames,
-                        scene_frame_xforms, cropped_frames);
+  // Store the "crop from" location on the input frame for use with an external
+  // renderer.
+  for (int i = 0; i < num_scene_frames; i++) {
+    const int left = scene_frame_xforms[i].at<float>(0, 2);
+    const int right = left + crop_width;
+    const int top = top_static_border_size;
+    const int bottom =
+        top_static_border_size +
+        (crop_height - top_static_border_size - bottom_static_border_size);
+    crop_from_location->push_back(
+        cv::Rect(left, top, right - left, bottom - top));
+  }
+
+  return AffineRetarget(cv::Size(crop_width, crop_height),
+                        scene_frames_or_empty, scene_frame_xforms,
+                        cropped_frames);
 }

 }  // namespace autoflip
@@ -48,14 +48,19 @@ class SceneCropper {
  SceneCropper() {}
  ~SceneCropper() {}

-  // Crops scene frames given SceneKeyFrameCropSummary, FocusPointFrames, and
-  // any prior FocusPointFrames (to ensure smoothness when there was no actual
-  // scene change).
+  // Computes transformation matrix given SceneKeyFrameCropSummary,
+  // FocusPointFrames, and any prior FocusPointFrames (to ensure smoothness when
+  // there was no actual scene change). Optionally crops the input frames based
+  // on the transform matrix if |cropped_frames| is not nullptr and
+  // |scene_frames_or_empty| isn't empty.
+  // TODO: split this function into two separate functions.
  ::mediapipe::Status CropFrames(
-      const SceneKeyFrameCropSummary& scene_summary,
-      const std::vector<cv::Mat>& scene_frames,
+      const SceneKeyFrameCropSummary& scene_summary, const int num_scene_frames,
+      const std::vector<cv::Mat>& scene_frames_or_empty,
      const std::vector<FocusPointFrame>& focus_point_frames,
      const std::vector<FocusPointFrame>& prior_focus_point_frames,
+      int top_static_border_size, int bottom_static_border_size,
+      std::vector<cv::Rect>* all_scene_frame_xforms,
      std::vector<cv::Mat>* cropped_frames) const;
 };

@@ -71,24 +71,16 @@ std::vector<FocusPointFrame> GetDefaultFocusPointFrames() {
  return GetFocusPointFrames(kNumSceneFrames);
 }

-// Checks that CropFrames checks output pointer is not null.
-TEST(SceneCropperTest, CropFramesChecksOutputNotNull) {
-  SceneCropper scene_cropper;
-  const auto status = scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), nullptr);
-  EXPECT_FALSE(status.ok());
-  EXPECT_THAT(status.ToString(), HasSubstr("Output cropped frames is null."));
-}
-
 // Checks that CropFrames checks that scene frames size is positive.
 TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) {
  SceneCropper scene_cropper;
  std::vector<cv::Mat> scene_frames(0);
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
  const auto status = scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), scene_frames,
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames);
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
  EXPECT_FALSE(status.ok());
  EXPECT_THAT(status.ToString(), HasSubstr("No scene frames."));
 }
@@ -97,10 +89,12 @@ TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) {
 TEST(SceneCropperTest, CropFramesChecksFocusPointFramesSize) {
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  const auto status = scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0),
-      &cropped_frames);
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
  EXPECT_FALSE(status.ok());
  EXPECT_THAT(status.ToString(), HasSubstr("Wrong size of FocusPointFrames"));
 }
@@ -111,9 +105,12 @@ TEST(SceneCropperTest, CropFramesChecksCropSizePositive) {
  scene_summary.set_crop_window_width(-1);
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  const auto status = scene_cropper.CropFrames(
-      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
-      GetFocusPointFrames(0), &cropped_frames);
+      scene_summary, scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
  EXPECT_FALSE(status.ok());
  EXPECT_THAT(status.ToString(), HasSubstr("Crop width is non-positive."));
 }
@@ -124,9 +121,12 @@ TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
  scene_summary.set_crop_window_height(kSceneHeight + 1);
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  const auto status = scene_cropper.CropFrames(
-      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
-      GetFocusPointFrames(0), &cropped_frames);
+      scene_summary, scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
  EXPECT_FALSE(status.ok());
  EXPECT_THAT(status.ToString(),
              HasSubstr("Crop height exceeds frame height."));
@@ -136,9 +136,12 @@ TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
 TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  MP_ASSERT_OK(scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames));
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames));
  ASSERT_EQ(cropped_frames.size(), kNumSceneFrames);
  for (int i = 0; i < kNumSceneFrames; ++i) {
    EXPECT_EQ(cropped_frames[i].rows, kCropHeight);
@@ -150,9 +153,12 @@ TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
 TEST(SceneCropperTest, CropFramesWorksWithPriorFocusPointFrames) {
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  MP_EXPECT_OK(scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(3), &cropped_frames));
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(3), 0, 0,
+      &crop_from_locations, &cropped_frames));
  EXPECT_EQ(cropped_frames.size(), kNumSceneFrames);
  for (int i = 0; i < kNumSceneFrames; ++i) {
    EXPECT_EQ(cropped_frames[i].rows, kCropHeight);
@@ -0,0 +1,42 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "face_mesh_tflite",
+    deps = [
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/graphs/face_mesh:desktop_calculators",
+    ],
+)
+
+cc_binary(
+    name = "face_mesh_cpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/face_mesh:desktop_live_calculators",
+    ],
+)
+
+# Linux only
+cc_binary(
+    name = "face_mesh_gpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
+        "//mediapipe/graphs/face_mesh:desktop_live_gpu_calculators",
+    ],
+)
@@ -0,0 +1,21 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow *window;
+
+@end
@@ -0,0 +1,59 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+
+- (BOOL)application:(UIApplication *)application
+    didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+  // Override point for customization after application launch.
+  return YES;
+}
+
+- (void)applicationWillResignActive:(UIApplication *)application {
+  // Sent when the application is about to move from active to inactive state. This can occur for
+  // certain types of temporary interruptions (such as an incoming phone call or SMS message) or
+  // when the user quits the application and it begins the transition to the background state. Use
+  // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering
+  // callbacks. Games should use this method to pause the game.
+}
+
+- (void)applicationDidEnterBackground:(UIApplication *)application {
+  // Use this method to release shared resources, save user data, invalidate timers, and store
+  // enough application state information to restore your application to its current state in case
+  // it is terminated later. If your application supports background execution, this method is
+  // called instead of applicationWillTerminate: when the user quits.
+}
+
+- (void)applicationWillEnterForeground:(UIApplication *)application {
+  // Called as part of the transition from the background to the active state; here you can undo
+  // many of the changes made on entering the background.
+}
+
+- (void)applicationDidBecomeActive:(UIApplication *)application {
+  // Restart any tasks that were paused (or not yet started) while the application was inactive. If
+  // the application was previously in the background, optionally refresh the user interface.
+}
+
+- (void)applicationWillTerminate:(UIApplication *)application {
+  // Called when the application is about to terminate. Save data if appropriate. See also
+  // applicationDidEnterBackground:.
+}
+
+@end
@@ -0,0 +1,99 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "83.5x83.5",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "size" : "1024x1024",
+      "scale" : "1x"
+    }
+  ],
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
+
@@ -0,0 +1,7 @@
+{
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
+
@@ -0,0 +1,76 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "@build_bazel_rules_apple//apple:ios.bzl",
+    "ios_application",
+)
+
+licenses(["notice"])  # Apache 2.0
+
+MIN_IOS_VERSION = "10.0"
+
+ios_application(
+    name = "FaceMeshGpuApp",
+    bundle_id = "com.google.mediapipe.FaceMeshGpu",
+    families = [
+        "iphone",
+        "ipad",
+    ],
+    infoplists = ["Info.plist"],
+    minimum_os_version = MIN_IOS_VERSION,
+    provisioning_profile = "//mediapipe/examples/ios:provisioning_profile",
+    deps = [
+        ":FaceMeshGpuAppLibrary",
+        "@ios_opencv//:OpencvFramework",
+    ],
+)
+
+objc_library(
+    name = "FaceMeshGpuAppLibrary",
+    srcs = [
+        "AppDelegate.m",
+        "ViewController.mm",
+        "main.m",
+    ],
+    hdrs = [
+        "AppDelegate.h",
+        "ViewController.h",
+    ],
+    data = [
+        "Base.lproj/LaunchScreen.storyboard",
+        "Base.lproj/Main.storyboard",
+        "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu_binary_graph",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+    ],
+    sdk_frameworks = [
+        "AVFoundation",
+        "CoreGraphics",
+        "CoreMedia",
+        "UIKit",
+    ],
+    deps = [
+        "//mediapipe/objc:mediapipe_framework_ios",
+        "//mediapipe/objc:mediapipe_input_sources_ios",
+        "//mediapipe/objc:mediapipe_layer_renderer",
+    ] + select({
+        "//mediapipe:ios_i386": [],
+        "//mediapipe:ios_x86_64": [],
+        "//conditions:default": [
+            "//mediapipe/graphs/face_mesh:mobile_calculators",
+            "//mediapipe/framework/formats:landmark_cc_proto",
+        ],
+    }),
+)
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14490.70" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14490.49"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="ViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
+                                <rect key="frame" x="0.0" y="20" width="375" height="647"/>
+                                <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                                <subviews>
+                                    <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
+                                        <rect key="frame" x="57" y="248" width="260" height="151"/>
+                                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
+                                        <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                        <color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <nil key="highlightedColor"/>
+                                    </label>
+                                </subviews>
+                                <color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                <accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
+                                    <bool key="isElement" value="YES"/>
+                                </accessibility>
+                            </view>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                    <connections>
+                        <outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
+                        <outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="48.799999999999997" y="20.239880059970016"/>
+        </scene>
+    </scenes>
+</document>
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>NSCameraUsageDescription</key>
+  <string>This app uses the camera to demonstrate live video processing.</string>
+  <key>CFBundleDevelopmentRegion</key>
+  <string>en</string>
+  <key>CFBundleExecutable</key>
+  <string>$(EXECUTABLE_NAME)</string>
+  <key>CFBundleIdentifier</key>
+  <string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+  <key>CFBundleInfoDictionaryVersion</key>
+  <string>6.0</string>
+  <key>CFBundleName</key>
+  <string>$(PRODUCT_NAME)</string>
+  <key>CFBundlePackageType</key>
+  <string>APPL</string>
+  <key>CFBundleShortVersionString</key>
+  <string>1.0</string>
+  <key>CFBundleVersion</key>
+  <string>1</string>
+  <key>LSRequiresIPhoneOS</key>
+  <true/>
+  <key>UILaunchStoryboardName</key>
+  <string>LaunchScreen</string>
+  <key>UIMainStoryboardFile</key>
+  <string>Main</string>
+  <key>UIRequiredDeviceCapabilities</key>
+  <array>
+    <string>armv7</string>
+  </array>
+  <key>UISupportedInterfaceOrientations</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+  <key>UISupportedInterfaceOrientations~ipad</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+</dict>
+</plist>
@@ -0,0 +1,19 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface ViewController : UIViewController
+
+@end
@@ -0,0 +1,210 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "ViewController.h"
+
+#import "mediapipe/objc/MPPCameraInputSource.h"
+#import "mediapipe/objc/MPPGraph.h"
+#import "mediapipe/objc/MPPLayerRenderer.h"
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+static NSString* const kGraphName = @"face_mesh_mobile_gpu";
+
+static const char* kInputStream = "input_video";
+static const char* kNumFacesInputSidePacket = "num_faces";
+static const char* kOutputStream = "output_video";
+static const char* kLandmarksOutputStream = "multi_face_landmarks";
+static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
+
+// Max number of faces to detect/process.
+static const int kNumFaces = 1;
+
+@interface ViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
+
+// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and
+// sent video frames on _videoQueue.
+@property(nonatomic) MPPGraph* mediapipeGraph;
+
+@end
+
+@implementation ViewController {
+  /// Handles camera access via AVCaptureSession library.
+  MPPCameraInputSource* _cameraSource;
+
+  /// Inform the user when camera is unavailable.
+  IBOutlet UILabel* _noCameraLabel;
+  /// Display the camera preview frames.
+  IBOutlet UIView* _liveView;
+  /// Render frames in a layer.
+  MPPLayerRenderer* _renderer;
+
+  /// Process camera frames on this queue.
+  dispatch_queue_t _videoQueue;
+}
+
+#pragma mark - Cleanup methods
+
+- (void)dealloc {
+  self.mediapipeGraph.delegate = nil;
+  [self.mediapipeGraph cancel];
+  // Ignore errors since we're cleaning up.
+  [self.mediapipeGraph closeAllInputStreamsWithError:nil];
+  [self.mediapipeGraph waitUntilDoneWithError:nil];
+}
+
+#pragma mark - MediaPipe graph methods
+
+ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
+  // Load the graph config resource.
+  NSError* configLoadError = nil;
+  NSBundle* bundle = [NSBundle bundleForClass:[self class]];
+  if (!resource || resource.length == 0) {
+    return nil;
+  }
+  NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
+  NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
+  if (!data) {
+    NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError);
+    return nil;
+  }
+
+  // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object.
+  mediapipe::CalculatorGraphConfig config;
+  config.ParseFromArray(data.bytes, data.length);
+
+  // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
+  MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
+  [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
+  [newGraph addFrameOutputStream:kLandmarksOutputStream outputPacketType:MPPPacketTypeRaw];
+  [newGraph setSidePacket:(mediapipe::MakePacket<int>(kNumFaces)) named:kNumFacesInputSidePacket];
+  return newGraph;
+}
+
+#pragma mark - UIViewController methods
+
+- (void)viewDidLoad {
+  [super viewDidLoad];
+
+  _renderer = [[MPPLayerRenderer alloc] init];
+  _renderer.layer.frame = _liveView.layer.bounds;
+  [_liveView.layer addSublayer:_renderer.layer];
+  _renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop;
+  // When using the front camera, mirror the input for a more natural look.
+  _renderer.mirrored = YES;
+
+  dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class(
+      DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0);
+  _videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute);
+
+  _cameraSource = [[MPPCameraInputSource alloc] init];
+  [_cameraSource setDelegate:self queue:_videoQueue];
+  _cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
+  _cameraSource.cameraPosition = AVCaptureDevicePositionFront;
+  // The frame's native format is rotated with respect to the portrait orientation.
+  _cameraSource.orientation = AVCaptureVideoOrientationPortrait;
+
+  self.mediapipeGraph = [[self class] loadGraphFromResource:kGraphName];
+  self.mediapipeGraph.delegate = self;
+  // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
+  self.mediapipeGraph.maxFramesInFlight = 2;
+}
+
+// In this application, there is only one ViewController which has no navigation to other view
+// controllers, and there is only one View with live display showing the result of running the
+// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph
+// setup/teardown and camera start/stop logic should be updated appropriately in response to the
+// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times
+// depending on the application navigation flow in that case.
+- (void)viewWillAppear:(BOOL)animated {
+  [super viewWillAppear:animated];
+
+  [_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
+    if (granted) {
+      [self startGraphAndCamera];
+      dispatch_async(dispatch_get_main_queue(), ^{
+        _noCameraLabel.hidden = YES;
+      });
+    }
+  }];
+}
+
+- (void)startGraphAndCamera {
+  // Start running self.mediapipeGraph.
+  NSError* error;
+  if (![self.mediapipeGraph startWithError:&error]) {
+    NSLog(@"Failed to start graph: %@", error);
+  }
+
+  // Start fetching frames from the camera.
+  dispatch_async(_videoQueue, ^{
+    [_cameraSource start];
+  });
+}
+
+#pragma mark - MPPGraphDelegate methods
+
+// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+    didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
+              fromStream:(const std::string&)streamName {
+  if (streamName == kOutputStream) {
+    // Display the captured image on the screen.
+    CVPixelBufferRetain(pixelBuffer);
+    dispatch_async(dispatch_get_main_queue(), ^{
+      [_renderer renderPixelBuffer:pixelBuffer];
+      CVPixelBufferRelease(pixelBuffer);
+    });
+  }
+}
+
+// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+     didOutputPacket:(const ::mediapipe::Packet&)packet
+          fromStream:(const std::string&)streamName {
+  if (streamName == kLandmarksOutputStream) {
+    if (packet.IsEmpty()) {
+      NSLog(@"[TS:%lld] No face landmarks", packet.Timestamp().Value());
+      return;
+    }
+    const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
+    NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(),
+          multi_face_landmarks.size());
+    for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) {
+      const auto& landmarks = multi_face_landmarks[face_index];
+      NSLog(@"\tNumber of landmarks for face[%d]: %d", face_index, landmarks.landmark_size());
+      for (int i = 0; i < landmarks.landmark_size(); ++i) {
+        NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(),
+              landmarks.landmark(i).y(), landmarks.landmark(i).z());
+      }
+    }
+  }
+}
+
+#pragma mark - MPPInputSourceDelegate methods
+
+// Must be invoked on _videoQueue.
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
+                timestamp:(CMTime)timestamp
+               fromSource:(MPPInputSource*)source {
+  if (source != _cameraSource) {
+    NSLog(@"Unknown source: %@", source);
+    return;
+  }
+  [self.mediapipeGraph sendPixelBuffer:imageBuffer
+                            intoStream:kInputStream
+                            packetType:MPPPacketTypePixelBuffer];
+}
+
+@end
@@ -0,0 +1,22 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char * argv[]) {
+  @autoreleasepool {
+      return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
+  }
+}