Project import generated by Copybara.

GitOrigin-RevId: d0039a576e2db9c0fcefffd26a527df74cbe145b

README.md

@@ -9,26 +9,34 @@
 
 ## ML Solutions in MediaPipe
 
-* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
+* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [(web demo)](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
+* [Face Mesh](mediapipe/docs/face_mesh_mobile_gpu.md)
+* [Hand Detection](mediapipe/docs/hand_detection_mobile_gpu.md)
+* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [(web demo)](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
 * [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
-* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
-* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
+* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [(web demo)](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
 * [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
 * [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
 * [Objectron: 3D Object Detection and Tracking](mediapipe/docs/objectron_mobile_gpu.md)
-* [AutoFlip](mediapipe/docs/autoflip.md)
+* [AutoFlip: Intelligent Video Reframing](mediapipe/docs/autoflip.md)
+* [KNIFT: Template Matching with Neural Image Features](mediapipe/docs/template_matching_mobile_cpu.md)
 
 ![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
-![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
-![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
+![face_mesh](mediapipe/docs/images/mobile/face_mesh_android_gpu_small.gif)
+![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_android_gpu_small.gif)
+![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_3d_android_gpu_small.gif)
 ![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
+![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif)
 ![object_tracking](mediapipe/docs/images/mobile/object_tracking_android_gpu_small.gif)
+![objectron_shoes](mediapipe/docs/images/mobile/objectron_shoe_android_gpu_small.gif)
+![objectron_chair](mediapipe/docs/images/mobile/objectron_chair_android_gpu_small.gif)
+![template_matching](mediapipe/docs/images/mobile/template_matching_android_cpu_small.gif)
 
 ## Installation
 Follow these [instructions](mediapipe/docs/install.md).
 
 ## Getting started
-See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).
+See mobile, desktop, web and Google Coral [examples](mediapipe/docs/examples.md).
 
 Check out some web demos [[Edge detection]](https://viz.mediapipe.dev/runner/demos/edge_detection/edge_detection.html) [[Face detection]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html) [[Hand Tracking]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
 
@@ -40,10 +48,15 @@ Check out the [Examples page](https://mediapipe.readthedocs.io/en/latest/example
 ## Visualizing MediaPipe graphs
 A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.dev/). Please also see instructions [here](mediapipe/docs/visualizer.md).
 
+## Google Open Source Code search
+Search MediaPipe Github repository using [Google Open Source code search](https://t.co/LSZnbMUUnT?amp=1)
+
 ## Videos
 *  [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)
 
 ## Publications
+* [MediaPipe KNIFT: Template-based Feature Matching](https://mediapipe.page.link/knift-blog)
+* [Alfred Camera: Smart camera features using MediaPipe](https://developers.googleblog.com/2020/03/alfred-camera-smart-camera-features-using-mediapipe.html)
 * [MediaPipe Objectron: Real-time 3D Object Detection on Mobile Devices](https://mediapipe.page.link/objectron-aiblog)
 * [AutoFlip: An Open Source Framework for Intelligent Video Reframing](https://mediapipe.page.link/autoflip)
 * [Google Developer Blog: MediaPipe on the Web](https://mediapipe.page.link/webdevblog)
@@ -52,6 +65,7 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
 * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
 
 ## Events
+* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
 * [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
 * [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
 * [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -184,23 +184,14 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
   use_local_timestamp_ = spectrogram_options.use_local_timestamp();
 
   if (spectrogram_options.frame_duration_seconds() <= 0.0) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
-        << "Invalid or missing frame_duration_seconds.\n"
-           "frame_duration_seconds: "
-        << spectrogram_options.frame_overlap_seconds();
+    // TODO: return an error.
   }
   if (spectrogram_options.frame_overlap_seconds() >=
       spectrogram_options.frame_duration_seconds()) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
-        << "Invalid frame_overlap_seconds.\nframe_overlap_seconds: "
-        << spectrogram_options.frame_overlap_seconds()
-        << "\nframe_duration_seconds: "
-        << spectrogram_options.frame_duration_seconds();
+    // TODO: return an error.
   }
   if (spectrogram_options.frame_overlap_seconds() < 0.0) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
-        << "Frame_overlap_seconds is < 0.0.\nframe_overlap_seconds: "
-        << spectrogram_options.frame_overlap_seconds();
+    // TODO: return an error.
   }
 
   TimeSeriesHeader input_header;
@@ -212,9 +203,7 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
   if (!spectrogram_options.allow_multichannel_input() &&
       num_input_channels_ != 1) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
-        << "The current setting only supports single-channel input. Please set "
-           "allow_multichannel_input.\n";
+    // TODO: return an error.
   }
 
   frame_duration_samples_ =
@@ -293,10 +282,7 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
   const Matrix& input_stream = cc->Inputs().Index(0).Get<Matrix>();
   if (input_stream.rows() != num_input_channels_) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
-        << "Number of input channels do not correspond to the number of rows "
-        << "in the input matrix: " << num_input_channels_ << "channels vs "
-        << input_stream.rows() << " rows";
+    // TODO: return an error.
   }
 
   cumulative_input_samples_ += input_stream.cols();

mediapipe/calculators/core/BUILD

@@ -815,6 +815,38 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "split_normalized_landmark_list_calculator",
+    srcs = ["split_normalized_landmark_list_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":split_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "split_normalized_landmark_list_calculator_test",
+    srcs = ["split_normalized_landmark_list_calculator_test.cc"],
+    deps = [
+        ":split_normalized_landmark_list_calculator",
+        ":split_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:validate_type",
+    ],
+)
+
 cc_library(
     name = "dequantize_byte_array_calculator",
     srcs = ["dequantize_byte_array_calculator.cc"],

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -51,8 +51,8 @@ namespace mediapipe {
 class ConstantSidePacketCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    const auto& options = cc->Options().GetExtension(
-        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+    const auto& options =
+        cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
     RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag),
                  options.packet_size())
         << "Number of output side packets has to be same as number of packets "
@@ -80,8 +80,8 @@ class ConstantSidePacketCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) override {
-    const auto& options = cc->Options().GetExtension(
-        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+    const auto& options =
+        cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
     int index = 0;
     for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
          id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {

mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc

@@ -0,0 +1,165 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_  // NOLINT
+#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_  // NOLINT
+
+#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/resource_util.h"
+
+namespace mediapipe {
+
+// Splits an input packet with NormalizedLandmarkList into
+// multiple NormalizedLandmarkList output packets using the [begin, end) ranges
+// specified in SplitVectorCalculatorOptions. If the option "element_only" is
+// set to true, all ranges should be of size 1 and all outputs will be elements
+// of type NormalizedLandmark. If "element_only" is false, ranges can be
+// non-zero in size and all outputs will be of type NormalizedLandmarkList.
+// If the option "combine_outputs" is set to true, only one output stream can be
+// specified and all ranges of elements will be combined into one
+// NormalizedLandmarkList.
+class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().NumEntries() == 1);
+    RET_CHECK(cc->Outputs().NumEntries() != 0);
+
+    cc->Inputs().Index(0).Set<NormalizedLandmarkList>();
+
+    const auto& options =
+        cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
+
+    if (options.combine_outputs()) {
+      RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
+      cc->Outputs().Index(0).Set<NormalizedLandmarkList>();
+      for (int i = 0; i < options.ranges_size() - 1; ++i) {
+        for (int j = i + 1; j < options.ranges_size(); ++j) {
+          const auto& range_0 = options.ranges(i);
+          const auto& range_1 = options.ranges(j);
+          if ((range_0.begin() >= range_1.begin() &&
+               range_0.begin() < range_1.end()) ||
+              (range_1.begin() >= range_0.begin() &&
+               range_1.begin() < range_0.end())) {
+            return ::mediapipe::InvalidArgumentError(
+                "Ranges must be non-overlapping when using combine_outputs "
+                "option.");
+          }
+        }
+      }
+    } else {
+      if (cc->Outputs().NumEntries() != options.ranges_size()) {
+        return ::mediapipe::InvalidArgumentError(
+            "The number of output streams should match the number of ranges "
+            "specified in the CalculatorOptions.");
+      }
+
+      // Set the output types for each output stream.
+      for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
+        if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 ||
+            options.ranges(i).begin() >= options.ranges(i).end()) {
+          return ::mediapipe::InvalidArgumentError(
+              "Indices should be non-negative and begin index should be less "
+              "than the end index.");
+        }
+        if (options.element_only()) {
+          if (options.ranges(i).end() - options.ranges(i).begin() != 1) {
+            return ::mediapipe::InvalidArgumentError(
+                "Since element_only is true, all ranges should be of size 1.");
+          }
+          cc->Outputs().Index(i).Set<NormalizedLandmark>();
+        } else {
+          cc->Outputs().Index(i).Set<NormalizedLandmarkList>();
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+
+    const auto& options =
+        cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
+
+    element_only_ = options.element_only();
+    combine_outputs_ = options.combine_outputs();
+
+    for (const auto& range : options.ranges()) {
+      ranges_.push_back({range.begin(), range.end()});
+      max_range_end_ = std::max(max_range_end_, range.end());
+      total_elements_ += range.end() - range.begin();
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    const NormalizedLandmarkList& input =
+        cc->Inputs().Index(0).Get<NormalizedLandmarkList>();
+    RET_CHECK_GE(input.landmark_size(), max_range_end_);
+
+    if (combine_outputs_) {
+      NormalizedLandmarkList output;
+      for (int i = 0; i < ranges_.size(); ++i) {
+        for (int j = ranges_[i].first; j < ranges_[i].second; ++j) {
+          const NormalizedLandmark& input_landmark = input.landmark(j);
+          *output.add_landmark() = input_landmark;
+        }
+      }
+      RET_CHECK_EQ(output.landmark_size(), total_elements_);
+      cc->Outputs().Index(0).AddPacket(
+          MakePacket<NormalizedLandmarkList>(output).At(cc->InputTimestamp()));
+    } else {
+      if (element_only_) {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<NormalizedLandmark>(input.landmark(ranges_[i].first))
+                  .At(cc->InputTimestamp()));
+        }
+      } else {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          NormalizedLandmarkList output;
+          for (int j = ranges_[i].first; j < ranges_[i].second; ++j) {
+            const NormalizedLandmark& input_landmark = input.landmark(j);
+            *output.add_landmark() = input_landmark;
+          }
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<NormalizedLandmarkList>(output).At(
+                  cc->InputTimestamp()));
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  std::vector<std::pair<int32, int32>> ranges_;
+  int32 max_range_end_ = -1;
+  int32 total_elements_ = 0;
+  bool element_only_ = false;
+  bool combine_outputs_ = false;
+};
+
+REGISTER_CALCULATOR(SplitNormalizedLandmarkListCalculator);
+
+}  // namespace mediapipe
+
+// NOLINTNEXTLINE
+#endif  // MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_

mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc

@@ -0,0 +1,404 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+
+constexpr float kLocationVal = 3;
+
+class SplitNormalizedLandmarkListCalculatorTest : public ::testing::Test {
+ protected:
+  void TearDown() { expected_landmarks_.reset(); }
+
+  void PrepareNormalizedLandmarkList(int list_size) {
+    // Prepare input landmark list.
+    input_landmarks_ = absl::make_unique<NormalizedLandmarkList>();
+    expected_landmarks_ = absl::make_unique<NormalizedLandmarkList>();
+    for (int i = 0; i < list_size; ++i) {
+      NormalizedLandmark* landmark = input_landmarks_->add_landmark();
+      landmark->set_x(i * kLocationVal);
+      landmark->set_y(i * kLocationVal);
+      landmark->set_z(i * kLocationVal);
+      // Save the landmarks for comparison after the graph runs.
+      *expected_landmarks_->add_landmark() = *landmark;
+    }
+  }
+
+  void ValidateListOutput(std::vector<Packet>& output_packets,
+                          int expected_elements, int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+    const NormalizedLandmarkList& output_landmarks =
+        output_packets[0].Get<NormalizedLandmarkList>();
+    ASSERT_EQ(expected_elements, output_landmarks.landmark_size());
+
+    for (int i = 0; i < expected_elements; ++i) {
+      const NormalizedLandmark& expected_landmark =
+          expected_landmarks_->landmark(input_begin_index + i);
+      const NormalizedLandmark& result = output_landmarks.landmark(i);
+      EXPECT_FLOAT_EQ(expected_landmark.x(), result.x());
+      EXPECT_FLOAT_EQ(expected_landmark.y(), result.y());
+      EXPECT_FLOAT_EQ(expected_landmark.z(), result.z());
+    }
+  }
+
+  void ValidateCombinedListOutput(std::vector<Packet>& output_packets,
+                                  int expected_elements,
+                                  std::vector<int>& input_begin_indices,
+                                  std::vector<int>& input_end_indices) {
+    ASSERT_EQ(1, output_packets.size());
+    ASSERT_EQ(input_begin_indices.size(), input_end_indices.size());
+    const NormalizedLandmarkList& output_landmarks =
+        output_packets[0].Get<NormalizedLandmarkList>();
+    ASSERT_EQ(expected_elements, output_landmarks.landmark_size());
+    const int num_ranges = input_begin_indices.size();
+
+    int element_id = 0;
+    for (int range_id = 0; range_id < num_ranges; ++range_id) {
+      for (int i = input_begin_indices[range_id];
+           i < input_end_indices[range_id]; ++i) {
+        const NormalizedLandmark& expected_landmark =
+            expected_landmarks_->landmark(i);
+        const NormalizedLandmark& result =
+            output_landmarks.landmark(element_id);
+        EXPECT_FLOAT_EQ(expected_landmark.x(), result.x());
+        EXPECT_FLOAT_EQ(expected_landmark.y(), result.y());
+        EXPECT_FLOAT_EQ(expected_landmark.z(), result.z());
+        element_id++;
+      }
+    }
+  }
+
+  void ValidateElementOutput(std::vector<Packet>& output_packets,
+                             int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+
+    const NormalizedLandmark& output_landmark =
+        output_packets[0].Get<NormalizedLandmark>();
+    ASSERT_TRUE(output_landmark.IsInitialized());
+
+    const NormalizedLandmark& expected_landmark =
+        expected_landmarks_->landmark(input_begin_index);
+
+    EXPECT_FLOAT_EQ(expected_landmark.x(), output_landmark.x());
+    EXPECT_FLOAT_EQ(expected_landmark.y(), output_landmark.y());
+    EXPECT_FLOAT_EQ(expected_landmark.z(), output_landmark.z());
+  }
+
+  std::unique_ptr<NormalizedLandmarkList> input_landmarks_ = nullptr;
+  std::unique_ptr<NormalizedLandmarkList> expected_landmarks_ = nullptr;
+  std::unique_ptr<CalculatorRunner> runner_ = nullptr;
+};
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
+  PrepareNormalizedLandmarkList(/*list_size=*/5);
+  ASSERT_NE(input_landmarks_, nullptr);
+
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ValidateListOutput(range_0_packets, /*expected_elements=*/1,
+                     /*input_begin_index=*/0);
+  ValidateListOutput(range_1_packets, /*expected_elements=*/3,
+                     /*input_begin_index=*/1);
+  ValidateListOutput(range_2_packets, /*expected_elements=*/1,
+                     /*input_begin_index=*/4);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("landmarks_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 0 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because of an invalid range (begin == end).
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       InvalidOutputStreamCountTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because the number of output streams does not
+  // match the number of range elements in the options.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       InvalidCombineOutputsMultipleOutputsTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because the number of output streams does not
+  // match the number of range elements in the options.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       InvalidOverlappingRangesTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 3 }
+                  ranges: { begin: 1 end: 4 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because there are overlapping ranges.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
+  PrepareNormalizedLandmarkList(/*list_size=*/5);
+  ASSERT_NE(input_landmarks_, nullptr);
+
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ValidateElementOutput(range_0_packets,
+                        /*input_begin_index=*/0);
+  ValidateElementOutput(range_1_packets,
+                        /*input_begin_index=*/2);
+  ValidateElementOutput(range_2_packets,
+                        /*input_begin_index=*/4);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("landmarks_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
+  PrepareNormalizedLandmarkList(/*list_size=*/5);
+  ASSERT_NE(input_landmarks_, nullptr);
+
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  std::vector<int> input_begin_indices = {0, 2, 4};
+  std::vector<int> input_end_indices = {1, 3, 5};
+  ValidateCombinedListOutput(range_0_packets, /*expected_elements=*/3,
+                             input_begin_indices, input_end_indices);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("landmarks_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       ElementOnlyDisablesVectorOutputs) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/image/image_properties_calculator.cc

@@ -19,6 +19,11 @@
 #include "mediapipe/gpu/gpu_buffer.h"
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
+namespace {
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+}  // namespace
+
 namespace mediapipe {
 
 // Extracts image properties from the input image and outputs the properties.
@@ -40,13 +45,14 @@ namespace mediapipe {
 class ImagePropertiesCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
-    if (cc->Inputs().HasTag("IMAGE")) {
-      cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+    RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
+              cc->Inputs().HasTag(kGpuBufferTag));
+    if (cc->Inputs().HasTag(kImageFrameTag)) {
+      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
     }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-    if (cc->Inputs().HasTag("IMAGE_GPU")) {
-      cc->Inputs().Tag("IMAGE_GPU").Set<::mediapipe::GpuBuffer>();
+    if (cc->Inputs().HasTag(kGpuBufferTag)) {
+      cc->Inputs().Tag(kGpuBufferTag).Set<::mediapipe::GpuBuffer>();
     }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
@@ -66,16 +72,17 @@ class ImagePropertiesCalculator : public CalculatorBase {
     int width;
     int height;
 
-    if (cc->Inputs().HasTag("IMAGE") && !cc->Inputs().Tag("IMAGE").IsEmpty()) {
-      const auto& image = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+    if (cc->Inputs().HasTag(kImageFrameTag) &&
+        !cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
+      const auto& image = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
       width = image.Width();
       height = image.Height();
     }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-    if (cc->Inputs().HasTag("IMAGE_GPU") &&
-        !cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
+    if (cc->Inputs().HasTag(kGpuBufferTag) &&
+        !cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
       const auto& image =
-          cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+          cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
       width = image.width();
       height = image.height();
     }

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -47,6 +47,9 @@ namespace mediapipe {
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
 namespace {
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+
 int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
   switch (rotation) {
     case mediapipe::RotationMode_Mode_UNKNOWN:
@@ -95,7 +98,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 // Scales, rotates, and flips images horizontally or vertically.
 //
 // Input:
-//   One of the following two tags:
+//   One of the following tags:
 //   IMAGE: ImageFrame representing the input image.
 //   IMAGE_GPU: GpuBuffer representing the input image.
 //
@@ -113,7 +116,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 //   corresponding field in the calculator options.
 //
 // Output:
-//   One of the following two tags:
+//   One of the following tags:
 //   IMAGE - ImageFrame representing the output image.
 //   IMAGE_GPU - GpuBuffer representing the output image.
 //
@@ -152,7 +155,8 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 // Note: To enable horizontal or vertical flipping, specify them in the
 // calculator options. Flipping is applied after rotation.
 //
-// Note: Only scale mode STRETCH is currently supported on CPU.
+// Note: Input defines output, so only matchig types supported:
+// IMAGE -> IMAGE  or  IMAGE_GPU -> IMAGE_GPU
 //
 class ImageTransformationCalculator : public CalculatorBase {
  public:
@@ -186,7 +190,7 @@ class ImageTransformationCalculator : public CalculatorBase {
 
   bool use_gpu_ = false;
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  GlCalculatorHelper helper_;
+  GlCalculatorHelper gpu_helper_;
   std::unique_ptr<QuadRenderer> rgb_renderer_;
   std::unique_ptr<QuadRenderer> yuv_renderer_;
   std::unique_ptr<QuadRenderer> ext_rgb_renderer_;
@@ -197,21 +201,22 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 // static
 ::mediapipe::Status ImageTransformationCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
-  RET_CHECK(cc->Outputs().HasTag("IMAGE") ^ cc->Outputs().HasTag("IMAGE_GPU"));
+  // Only one input can be set, and the output type must match.
+  RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
+            cc->Inputs().HasTag(kGpuBufferTag));
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("IMAGE")) {
-    RET_CHECK(cc->Outputs().HasTag("IMAGE"));
-    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
-    cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
+    RET_CHECK(cc->Outputs().HasTag(kImageFrameTag));
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+    cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
-    RET_CHECK(cc->Outputs().HasTag("IMAGE_GPU"));
-    cc->Inputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
-    cc->Outputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
+    RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag));
+    cc->Inputs().Tag(kGpuBufferTag).Set<GpuBuffer>();
+    cc->Outputs().Tag(kGpuBufferTag).Set<GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -259,7 +264,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
   options_ = cc->Options<ImageTransformationCalculatorOptions>();
 
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
     use_gpu_ = true;
   }
 
@@ -300,7 +305,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   if (use_gpu_) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
     // Let the helper access the GL context information.
-    MP_RETURN_IF_ERROR(helper_.Open(cc));
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
 #else
     RET_CHECK_FAIL() << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -328,18 +333,14 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
   if (use_gpu_) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-    if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
-      // Image is missing, hence no way to produce output image. (Timestamp
-      // bound will be updated automatically.)
+    if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
       return ::mediapipe::OkStatus();
     }
-    return helper_.RunInGlContext(
+    return gpu_helper_.RunInGlContext(
         [this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
-    if (cc->Inputs().Tag("IMAGE").IsEmpty()) {
-      // Image is missing, hence no way to produce output image. (Timestamp
-      // bound will be updated automatically.)
+    if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
       return ::mediapipe::OkStatus();
     }
     return RenderCpu(cc);
@@ -354,7 +355,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
     QuadRenderer* rgb_renderer = rgb_renderer_.release();
     QuadRenderer* yuv_renderer = yuv_renderer_.release();
     QuadRenderer* ext_rgb_renderer = ext_rgb_renderer_.release();
-    helper_.RunInGlContext([rgb_renderer, yuv_renderer, ext_rgb_renderer] {
+    gpu_helper_.RunInGlContext([rgb_renderer, yuv_renderer, ext_rgb_renderer] {
       if (rgb_renderer) {
         rgb_renderer->GlTeardown();
         delete rgb_renderer;
@@ -376,18 +377,21 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
 ::mediapipe::Status ImageTransformationCalculator::RenderCpu(
     CalculatorContext* cc) {
-  int input_width = cc->Inputs().Tag("IMAGE").Get<ImageFrame>().Width();
-  int input_height = cc->Inputs().Tag("IMAGE").Get<ImageFrame>().Height();
+  cv::Mat input_mat;
+  mediapipe::ImageFormat::Format format;
 
-  const auto& input_img = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
-  cv::Mat input_mat = formats::MatView(&input_img);
-  cv::Mat scaled_mat;
+  const auto& input = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
+  input_mat = formats::MatView(&input);
+  format = input.Format();
 
+  const int input_width = input_mat.cols;
+  const int input_height = input_mat.rows;
   if (!output_height_ || !output_width_) {
     output_height_ = input_height;
     output_width_ = input_width;
   }
 
+  cv::Mat scaled_mat;
   if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
     cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_));
   } else {
@@ -444,10 +448,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   }
 
   std::unique_ptr<ImageFrame> output_frame(
-      new ImageFrame(input_img.Format(), output_width, output_height));
+      new ImageFrame(format, output_width, output_height));
   cv::Mat output_mat = formats::MatView(output_frame.get());
   flipped_mat.copyTo(output_mat);
-  cc->Outputs().Tag("IMAGE").Add(output_frame.release(), cc->InputTimestamp());
+  cc->Outputs()
+      .Tag(kImageFrameTag)
+      .Add(output_frame.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
 }
@@ -455,8 +461,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 ::mediapipe::Status ImageTransformationCalculator::RenderGpu(
     CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  int input_width = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>().width();
-  int input_height = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>().height();
+  const auto& input = cc->Inputs().Tag(kGpuBufferTag).Get<GpuBuffer>();
+  const int input_width = input.width();
+  const int input_height = input.height();
 
   int output_width;
   int output_height;
@@ -472,7 +479,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
         .Add(padding.release(), cc->InputTimestamp());
   }
 
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>();
   QuadRenderer* renderer = nullptr;
   GlTexture src1;
 
@@ -486,11 +492,11 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
                                  {"video_frame_y", "video_frame_uv"}));
     }
     renderer = yuv_renderer_.get();
-    src1 = helper_.CreateSourceTexture(input, 0);
+    src1 = gpu_helper_.CreateSourceTexture(input, 0);
   } else  // NOLINT(readability/braces)
 #endif    // iOS
   {
-    src1 = helper_.CreateSourceTexture(input);
+    src1 = gpu_helper_.CreateSourceTexture(input);
 #if defined(TEXTURE_EXTERNAL_OES)
     if (src1.target() == GL_TEXTURE_EXTERNAL_OES) {
       if (!ext_rgb_renderer_) {
@@ -516,10 +522,10 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   mediapipe::FrameRotation rotation =
       mediapipe::FrameRotationFromDegrees(RotationModeToDegrees(rotation_));
 
-  auto dst = helper_.CreateDestinationTexture(output_width, output_height,
-                                              input.format());
+  auto dst = gpu_helper_.CreateDestinationTexture(output_width, output_height,
+                                                  input.format());
 
-  helper_.BindFramebuffer(dst);  // GL_TEXTURE0
+  gpu_helper_.BindFramebuffer(dst);  // GL_TEXTURE0
   glActiveTexture(GL_TEXTURE1);
   glBindTexture(src1.target(), src1.name());
 
@@ -534,8 +540,8 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   // Execute GL commands, before getting result.
   glFlush();
 
-  auto output = dst.GetFrame<GpuBuffer>();
-  cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
+  auto output = dst.template GetFrame<GpuBuffer>();
+  cc->Outputs().Tag(kGpuBufferTag).Add(output.release(), cc->InputTimestamp());
 
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 

mediapipe/calculators/image/recolor_calculator.cc

@@ -32,6 +32,11 @@
 
 namespace {
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
+
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kMaskCpuTag[] = "MASK";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+constexpr char kMaskGpuTag[] = "MASK_GPU";
 }  // namespace
 
 namespace mediapipe {
@@ -112,39 +117,41 @@ REGISTER_CALCULATOR(RecolorCalculator);
   bool use_gpu = false;
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
-    cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
+    cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Inputs().HasTag("IMAGE")) {
-    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag("MASK_GPU")) {
-    cc->Inputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
+  if (cc->Inputs().HasTag(kMaskGpuTag)) {
+    cc->Inputs().Tag(kMaskGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Inputs().HasTag("MASK")) {
-    cc->Inputs().Tag("MASK").Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kMaskCpuTag)) {
+    cc->Inputs().Tag(kMaskCpuTag).Set<ImageFrame>();
   }
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Outputs().HasTag("IMAGE_GPU")) {
-    cc->Outputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+  if (cc->Outputs().HasTag(kGpuBufferTag)) {
+    cc->Outputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Outputs().HasTag("IMAGE")) {
-    cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
+  if (cc->Outputs().HasTag(kImageFrameTag)) {
+    cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 
   // Confirm only one of the input streams is present.
-  RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
+            cc->Inputs().HasTag(kGpuBufferTag));
   // Confirm only one of the output streams is present.
-  RET_CHECK(cc->Outputs().HasTag("IMAGE") ^ cc->Outputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Outputs().HasTag(kImageFrameTag) ^
+            cc->Outputs().HasTag(kGpuBufferTag));
 
   if (use_gpu) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
@@ -158,7 +165,7 @@ REGISTER_CALCULATOR(RecolorCalculator);
 ::mediapipe::Status RecolorCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
     use_gpu_ = true;
 #if !defined(MEDIAPIPE_DISABLE_GPU)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
@@ -201,12 +208,12 @@ REGISTER_CALCULATOR(RecolorCalculator);
 }
 
 ::mediapipe::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
-  if (cc->Inputs().Tag("MASK").IsEmpty()) {
+  if (cc->Inputs().Tag(kMaskCpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
   // Get inputs and setup output.
-  const auto& input_img = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
-  const auto& mask_img = cc->Inputs().Tag("MASK").Get<ImageFrame>();
+  const auto& input_img = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
+  const auto& mask_img = cc->Inputs().Tag(kMaskCpuTag).Get<ImageFrame>();
 
   cv::Mat input_mat = formats::MatView(&input_img);
   cv::Mat mask_mat = formats::MatView(&mask_img);
@@ -254,19 +261,21 @@ REGISTER_CALCULATOR(RecolorCalculator);
     }
   }
 
-  cc->Outputs().Tag("IMAGE").Add(output_img.release(), cc->InputTimestamp());
+  cc->Outputs()
+      .Tag(kImageFrameTag)
+      .Add(output_img.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
-  if (cc->Inputs().Tag("MASK_GPU").IsEmpty()) {
+  if (cc->Inputs().Tag(kMaskGpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
   // Get inputs and setup output.
-  const Packet& input_packet = cc->Inputs().Tag("IMAGE_GPU").Value();
-  const Packet& mask_packet = cc->Inputs().Tag("MASK_GPU").Value();
+  const Packet& input_packet = cc->Inputs().Tag(kGpuBufferTag).Value();
+  const Packet& mask_packet = cc->Inputs().Tag(kMaskGpuTag).Value();
 
   const auto& input_buffer = input_packet.Get<mediapipe::GpuBuffer>();
   const auto& mask_buffer = mask_packet.Get<mediapipe::GpuBuffer>();
@@ -296,7 +305,7 @@ REGISTER_CALCULATOR(RecolorCalculator);
 
   // Send result image in GPU packet.
   auto output = dst_tex.GetFrame<mediapipe::GpuBuffer>();
-  cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kGpuBufferTag).Add(output.release(), cc->InputTimestamp());
 
   // Cleanup
   img_tex.Release();

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -244,7 +244,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
   ::mediapipe::Status VerifySequence() {
     std::string error_msg = "Missing features - ";
     bool all_present = true;
-    for (auto iter : features_present_) {
+    for (const auto& iter : features_present_) {
       if (!iter.second) {
         all_present = false;
         absl::StrAppend(&error_msg, iter.first, ", ");

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -126,7 +126,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase {
     // Set user specified tags properly.
     // If no tags specified will use tensorflow::kSavedModelTagServe by default.
     std::unordered_set<std::string> tags_set;
-    for (std::string tag : options.saved_model_tag()) {
+    for (const std::string& tag : options.saved_model_tag()) {
       tags_set.insert(tag);
     }
     if (tags_set.empty()) {

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -121,7 +121,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator {
     // Set user specified tags properly.
     // If no tags specified will use tensorflow::kSavedModelTagServe by default.
     std::unordered_set<std::string> tags_set;
-    for (std::string tag : options.saved_model_tag()) {
+    for (const std::string& tag : options.saved_model_tag()) {
       tags_set.insert(tag);
     }
     if (tags_set.empty()) {

mediapipe/calculators/tflite/BUILD

@@ -243,6 +243,7 @@ cc_library(
             "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
         ],
         "//conditions:default": [
+            "//mediapipe/util/tflite:tflite_gpu_runner",
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gpu_buffer",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",

mediapipe/calculators/tflite/tflite_converter_calculator.cc

@@ -63,6 +63,10 @@ typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
 typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
     ColMajorMatrixXf;
 
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
 }  // namespace
 
 namespace mediapipe {
@@ -124,6 +128,9 @@ struct GPUData {
 //  GPU tensors are currently only supported on mobile platforms.
 //  This calculator uses FixedSizeInputStreamHandler by default.
 //
+// Note: Input defines output, so only these type sets are supported:
+// IMAGE -> TENSORS | IMAGE_GPU -> TENSORS_GPU | MATRIX -> TENSORS
+//
 class TfLiteConverterCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
@@ -138,9 +145,9 @@ class TfLiteConverterCalculator : public CalculatorBase {
   template <class T>
   ::mediapipe::Status NormalizeImage(const ImageFrame& image_frame,
                                      bool zero_center, bool flip_vertically,
-                                     float* tensor_buffer);
+                                     float* tensor_ptr);
   ::mediapipe::Status CopyMatrixToTensor(const Matrix& matrix,
-                                         float* tensor_buffer);
+                                         float* tensor_ptr);
   ::mediapipe::Status ProcessCPU(CalculatorContext* cc);
   ::mediapipe::Status ProcessGPU(CalculatorContext* cc);
 
@@ -166,33 +173,35 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::GetContract(
     CalculatorContract* cc) {
-  const bool has_image_tag = cc->Inputs().HasTag("IMAGE");
-  const bool has_image_gpu_tag = cc->Inputs().HasTag("IMAGE_GPU");
-  const bool has_matrix_tag = cc->Inputs().HasTag("MATRIX");
   // Confirm only one of the input streams is present.
-  RET_CHECK(has_image_tag ^ has_image_gpu_tag ^ has_matrix_tag &&
-            !(has_image_tag && has_image_gpu_tag && has_matrix_tag));
+  RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
+            cc->Inputs().HasTag(kGpuBufferTag) ^ cc->Inputs().HasTag("MATRIX"));
 
   // Confirm only one of the output streams is present.
-  RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
-            cc->Outputs().HasTag("TENSORS_GPU"));
+  RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^
+            cc->Outputs().HasTag(kTensorsGpuTag));
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("IMAGE")) cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
-  if (cc->Inputs().HasTag("MATRIX")) cc->Inputs().Tag("MATRIX").Set<Matrix>();
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+  }
+  if (cc->Inputs().HasTag("MATRIX")) {
+    cc->Inputs().Tag("MATRIX").Set<Matrix>();
+  }
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
-    cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
+    cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-  if (cc->Outputs().HasTag("TENSORS"))
-    cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+  if (cc->Outputs().HasTag(kTensorsTag)) {
+    cc->Outputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
+  }
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Outputs().HasTag("TENSORS_GPU")) {
-    cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+  if (cc->Outputs().HasTag(kTensorsGpuTag)) {
+    cc->Outputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -216,8 +225,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
   MP_RETURN_IF_ERROR(LoadOptions(cc));
 
-  if (cc->Inputs().HasTag("IMAGE_GPU") ||
-      cc->Outputs().HasTag("IMAGE_OUT_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag) ||
+      cc->Outputs().HasTag(kGpuBufferTag)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     use_gpu_ = true;
 #else
@@ -227,8 +236,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
   if (use_gpu_) {
     // Cannot mix CPU/GPU streams.
-    RET_CHECK(cc->Inputs().HasTag("IMAGE_GPU") &&
-              cc->Outputs().HasTag("TENSORS_GPU"));
+    RET_CHECK(cc->Inputs().HasTag(kGpuBufferTag) &&
+              cc->Outputs().HasTag(kTensorsGpuTag));
     // Cannot use quantization.
     use_quantized_tensors_ = false;
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
@@ -248,7 +257,6 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::Process(CalculatorContext* cc) {
   if (use_gpu_) {
-    // GpuBuffer to tflite::gpu::GlBuffer conversion.
     if (!initialized_) {
       MP_RETURN_IF_ERROR(InitGpu(cc));
       initialized_ = true;
@@ -259,7 +267,6 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     // Convert to CPU tensors or Matrix type.
     MP_RETURN_IF_ERROR(ProcessCPU(cc));
   }
-
   return ::mediapipe::OkStatus();
 }
 
@@ -275,24 +282,26 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::ProcessCPU(
     CalculatorContext* cc) {
-  if (cc->Inputs().HasTag("IMAGE")) {
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
     // CPU ImageFrame to TfLiteTensor conversion.
 
-    const auto& image_frame = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+    const auto& image_frame =
+        cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
     const int height = image_frame.Height();
     const int width = image_frame.Width();
     const int channels = image_frame.NumberOfChannels();
     const int channels_preserved = std::min(channels, max_num_channels_);
+    const mediapipe::ImageFormat::Format format = image_frame.Format();
 
     if (!initialized_) {
-      if (!(image_frame.Format() == mediapipe::ImageFormat::SRGBA ||
-            image_frame.Format() == mediapipe::ImageFormat::SRGB ||
-            image_frame.Format() == mediapipe::ImageFormat::GRAY8 ||
-            image_frame.Format() == mediapipe::ImageFormat::VEC32F1))
+      if (!(format == mediapipe::ImageFormat::SRGBA ||
+            format == mediapipe::ImageFormat::SRGB ||
+            format == mediapipe::ImageFormat::GRAY8 ||
+            format == mediapipe::ImageFormat::VEC32F1))
         RET_CHECK_FAIL() << "Unsupported CPU input format.";
       TfLiteQuantization quant;
       if (use_quantized_tensors_) {
-        RET_CHECK(image_frame.Format() != mediapipe::ImageFormat::VEC32F1)
+        RET_CHECK(format != mediapipe::ImageFormat::VEC32F1)
             << "Only 8-bit input images are supported for quantization.";
         quant.type = kTfLiteAffineQuantization;
         quant.params = nullptr;
@@ -349,8 +358,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
     auto output_tensors = absl::make_unique<std::vector<TfLiteTensor>>();
     output_tensors->emplace_back(*tensor);
-    cc->Outputs().Tag("TENSORS").Add(output_tensors.release(),
-                                     cc->InputTimestamp());
+    cc->Outputs()
+        .Tag(kTensorsTag)
+        .Add(output_tensors.release(), cc->InputTimestamp());
   } else if (cc->Inputs().HasTag("MATRIX")) {
     // CPU Matrix to TfLiteTensor conversion.
 
@@ -371,15 +381,16 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     interpreter_->ResizeInputTensor(tensor_idx, {height, width, channels});
     interpreter_->AllocateTensors();
 
-    float* tensor_buffer = tensor->data.f;
-    RET_CHECK(tensor_buffer);
+    float* tensor_ptr = tensor->data.f;
+    RET_CHECK(tensor_ptr);
 
-    MP_RETURN_IF_ERROR(CopyMatrixToTensor(matrix, tensor_buffer));
+    MP_RETURN_IF_ERROR(CopyMatrixToTensor(matrix, tensor_ptr));
 
     auto output_tensors = absl::make_unique<std::vector<TfLiteTensor>>();
     output_tensors->emplace_back(*tensor);
-    cc->Outputs().Tag("TENSORS").Add(output_tensors.release(),
-                                     cc->InputTimestamp());
+    cc->Outputs()
+        .Tag(kTensorsTag)
+        .Add(output_tensors.release(), cc->InputTimestamp());
   }
 
   return ::mediapipe::OkStatus();
@@ -389,7 +400,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // GpuBuffer to tflite::gpu::GlBuffer conversion.
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+  const auto& input =
+      cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
   MP_RETURN_IF_ERROR(
       gpu_helper_.RunInGlContext([this, &input]() -> ::mediapipe::Status {
         // Convert GL texture into TfLite GlBuffer (SSBO).
@@ -421,11 +433,12 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
         return ::mediapipe::OkStatus();
       }));
   cc->Outputs()
-      .Tag("TENSORS_GPU")
+      .Tag(kTensorsGpuTag)
       .Add(output_tensors.release(), cc->InputTimestamp());
 #elif defined(MEDIAPIPE_IOS)
   // GpuBuffer to id<MTLBuffer> conversion.
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+  const auto& input =
+      cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
   id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
 
   id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
@@ -457,7 +470,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
                     commandBuffer:command_buffer];
 
   cc->Outputs()
-      .Tag("TENSORS_GPU")
+      .Tag(kTensorsGpuTag)
       .Add(output_tensors.release(), cc->InputTimestamp());
 #else
   RET_CHECK_FAIL() << "GPU processing is not enabled.";
@@ -469,7 +482,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 ::mediapipe::Status TfLiteConverterCalculator::InitGpu(CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
   // Get input image sizes.
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+  const auto& input =
+      cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
   mediapipe::ImageFormat::Format format =
       mediapipe::ImageFormatForGpuBufferFormat(input.format());
   gpu_data_out_ = absl::make_unique<GPUData>();
@@ -612,7 +626,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
   CHECK_LE(max_num_channels_, 4);
   CHECK_NE(max_num_channels_, 2);
 #if defined(MEDIAPIPE_IOS)
-  if (cc->Inputs().HasTag("IMAGE_GPU"))
+  if (cc->Inputs().HasTag(kGpuBufferTag))
     // Currently on iOS, tflite gpu input tensor must be 4 channels,
     // so input image must be 4 channels also (checked in InitGpu).
     max_num_channels_ = 4;
@@ -627,7 +641,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 template <class T>
 ::mediapipe::Status TfLiteConverterCalculator::NormalizeImage(
     const ImageFrame& image_frame, bool zero_center, bool flip_vertically,
-    float* tensor_buffer) {
+    float* tensor_ptr) {
   const int height = image_frame.Height();
   const int width = image_frame.Width();
   const int channels = image_frame.NumberOfChannels();
@@ -651,7 +665,7 @@ template <class T>
         (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
     for (int j = 0; j < width; ++j) {
       for (int c = 0; c < channels_preserved; ++c) {
-        *tensor_buffer++ = *image_ptr++ / div - sub;
+        *tensor_ptr++ = *image_ptr++ / div - sub;
       }
       image_ptr += channels_ignored;
     }
@@ -661,14 +675,14 @@ template <class T>
 }
 
 ::mediapipe::Status TfLiteConverterCalculator::CopyMatrixToTensor(
-    const Matrix& matrix, float* tensor_buffer) {
+    const Matrix& matrix, float* tensor_ptr) {
   if (row_major_matrix_) {
-    auto matrix_map = Eigen::Map<RowMajorMatrixXf>(tensor_buffer, matrix.rows(),
-                                                   matrix.cols());
+    auto matrix_map =
+        Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
     matrix_map = matrix;
   } else {
-    auto matrix_map = Eigen::Map<ColMajorMatrixXf>(tensor_buffer, matrix.rows(),
-                                                   matrix.cols());
+    auto matrix_map =
+        Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
     matrix_map = matrix;
   }
 

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -23,9 +23,9 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 
-#if !defined(__EMSCRIPTEN__)
+#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)
 #include "mediapipe/util/cpu_util.h"
-#endif  // !__EMSCRIPTEN__
+#endif  // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__
 
 #include "mediapipe/util/resource_util.h"
 #include "tensorflow/lite/error_reporter.h"
@@ -36,6 +36,7 @@
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gpu_buffer.h"
+#include "mediapipe/util/tflite/tflite_gpu_runner.h"
 #include "tensorflow/lite/delegates/gpu/common/shape.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_program.h"
@@ -75,6 +76,9 @@ typedef id<MTLBuffer> GpuTensor;
 
 // Round up n to next multiple of m.
 size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; }  // NOLINT
+
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
 }  // namespace
 
 #if defined(MEDIAPIPE_EDGE_TPU)
@@ -121,7 +125,7 @@ struct GPUData {
 
 // Returns number of threads to configure XNNPACK delegate with.
 // (Equal to user provided value if specified.  Otherwise, it returns number of
-// high cores (hard-coded to 1 for __EMSCRIPTEN__))
+// high cores (hard-coded to 1 for Emscripten without Threads extension))
 int GetXnnpackNumThreads(
     const mediapipe::TfLiteInferenceCalculatorOptions& opts) {
   static constexpr int kDefaultNumThreads = -1;
@@ -129,11 +133,11 @@ int GetXnnpackNumThreads(
       opts.delegate().xnnpack().num_threads() != kDefaultNumThreads) {
     return opts.delegate().xnnpack().num_threads();
   }
-#if !defined(__EMSCRIPTEN__)
+#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)
   return InferHigherCoreIds().size();
 #else
   return 1;
-#endif  // !__EMSCRIPTEN__
+#endif  // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__
 }
 
 // Calculator Header Section
@@ -219,6 +223,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   ::mediapipe::Status LoadModel(CalculatorContext* cc);
   ::mediapipe::StatusOr<Packet> GetModelAsPacket(const CalculatorContext& cc);
   ::mediapipe::Status LoadDelegate(CalculatorContext* cc);
+  ::mediapipe::Status InitTFLiteGPURunner();
 
   Packet model_packet_;
   std::unique_ptr<tflite::Interpreter> interpreter_;
@@ -228,6 +233,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
+  std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
 #elif defined(MEDIAPIPE_IOS)
   MPPMetalHelper* gpu_helper_ = nullptr;
   std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
@@ -245,6 +251,8 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   bool gpu_input_ = false;
   bool gpu_output_ = false;
   bool use_quantized_tensors_ = false;
+
+  bool use_advanced_gpu_api_ = false;
 };
 REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
@@ -252,10 +260,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
 ::mediapipe::Status TfLiteInferenceCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag("TENSORS") ^
-            cc->Inputs().HasTag("TENSORS_GPU"));
-  RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
-            cc->Outputs().HasTag("TENSORS_GPU"));
+  RET_CHECK(cc->Inputs().HasTag(kTensorsTag) ^
+            cc->Inputs().HasTag(kTensorsGpuTag));
+  RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^
+            cc->Outputs().HasTag(kTensorsGpuTag));
 
   const auto& options =
       cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
@@ -266,26 +274,26 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   bool use_gpu =
       options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
 
-  if (cc->Inputs().HasTag("TENSORS"))
-    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+  if (cc->Inputs().HasTag(kTensorsTag))
+    cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
     RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
         << "GPU input is compatible with GPU delegate only.";
 
-    cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Inputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-  if (cc->Outputs().HasTag("TENSORS"))
-    cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+  if (cc->Outputs().HasTag(kTensorsTag))
+    cc->Outputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Outputs().HasTag("TENSORS_GPU")) {
+  if (cc->Outputs().HasTag(kTensorsGpuTag)) {
     RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
         << "GPU output is compatible with GPU delegate only.";
 
-    cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Outputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -320,27 +328,31 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
   gpu_inference_ = options.use_gpu();
 
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     gpu_input_ = true;
     gpu_inference_ = true;  // Inference must be on GPU also.
 #else
-    RET_CHECK(!cc->Inputs().HasTag("TENSORS_GPU"))
+    RET_CHECK(!cc->Inputs().HasTag(kTensorsGpuTag))
         << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
-  if (cc->Outputs().HasTag("TENSORS_GPU")) {
+  if (cc->Outputs().HasTag(kTensorsGpuTag)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     gpu_output_ = true;
-    RET_CHECK(cc->Inputs().HasTag("TENSORS_GPU"))
+    RET_CHECK(cc->Inputs().HasTag(kTensorsGpuTag))
         << "GPU output must also have GPU Input.";
 #else
-    RET_CHECK(!cc->Inputs().HasTag("TENSORS_GPU"))
+    RET_CHECK(!cc->Inputs().HasTag(kTensorsGpuTag))
         << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
+  const auto& calculator_opts =
+      cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>();
+  use_advanced_gpu_api_ = false;
+
   MP_RETURN_IF_ERROR(LoadModel(cc));
 
   if (gpu_inference_) {
@@ -352,8 +364,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #endif
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
-        [this, &cc]() -> ::mediapipe::Status { return LoadDelegate(cc); }));
+    MP_RETURN_IF_ERROR(
+        gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
+          return use_advanced_gpu_api_ ? InitTFLiteGPURunner()
+                                       : LoadDelegate(cc);
+        }));
+    if (use_advanced_gpu_api_) return ::mediapipe::OkStatus();
 #else
     MP_RETURN_IF_ERROR(LoadDelegate(cc));
 #endif
@@ -365,13 +381,51 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status TfLiteInferenceCalculator::InitTFLiteGPURunner() {
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+  // Create and bind OpenGL buffers for outputs.
+  // These buffers are created onve and later their ids are jut passed to the
+  // calculator outputs.
+
+  gpu_data_out_.resize(tflite_gpu_runner_->outputs_size());
+  for (int i = 0; i < tflite_gpu_runner_->outputs_size(); ++i) {
+    gpu_data_out_[i] = absl::make_unique<GPUData>();
+    ASSIGN_OR_RETURN(gpu_data_out_[i]->elements,
+                     tflite_gpu_runner_->GetOutputElements(i));
+    // Create and bind input buffer.
+    RET_CHECK_CALL(::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
+        gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer));
+  }
+  RET_CHECK_CALL(tflite_gpu_runner_->Build());
+#endif
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status TfLiteInferenceCalculator::Process(CalculatorContext* cc) {
   // 1. Receive pre-processed tensor inputs.
-  if (gpu_input_) {
-    // Read GPU input into SSBO.
+  if (use_advanced_gpu_api_) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+    RET_CHECK(input_tensors.empty());
+    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
+        [this, &input_tensors]() -> ::mediapipe::Status {
+          for (int i = 0; i < input_tensors.size(); ++i) {
+            MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor(
+                input_tensors[i].id(), i));
+          }
+          for (int i = 0; i < gpu_data_out_.size(); ++i) {
+            MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToOutputTensor(
+                gpu_data_out_[i]->buffer.id(), i));
+          }
+          return ::mediapipe::OkStatus();
+        }));
+#endif
+  } else if (gpu_input_) {
+    // Read GPU input into SSBO.
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+    const auto& input_tensors =
+        cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
     RET_CHECK_GT(input_tensors.size(), 0);
     MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
         [this, &input_tensors]() -> ::mediapipe::Status {
@@ -386,7 +440,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
         }));
 #elif defined(MEDIAPIPE_IOS)
     const auto& input_tensors =
-        cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+        cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
     RET_CHECK_GT(input_tensors.size(), 0);
     // Explicit copy input with conversion float 32 bits to 16 bits.
     gpu_data_in_.resize(input_tensors.size());
@@ -413,7 +467,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   } else {
     // Read CPU input into tensors.
     const auto& input_tensors =
-        cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+        cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
     RET_CHECK_GT(input_tensors.size(), 0);
     for (int i = 0; i < input_tensors.size(); ++i) {
       const TfLiteTensor* input_tensor = &input_tensors[i];
@@ -437,7 +491,11 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this]() -> ::mediapipe::Status {
-          RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+          if (use_advanced_gpu_api_) {
+            RET_CHECK(tflite_gpu_runner_->Invoke().ok());
+          } else {
+            RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+          }
           return ::mediapipe::OkStatus();
         }));
 #elif defined(MEDIAPIPE_IOS)
@@ -448,7 +506,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   }
 
   // 3. Output processed tensors.
-  if (gpu_output_) {
+  if (use_advanced_gpu_api_) {
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+    auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
+    output_tensors->resize(gpu_data_out_.size());
+    for (int i = 0; i < gpu_data_out_.size(); ++i) {
+      output_tensors->at(i) = gpu_data_out_[0]->buffer.MakeRef();
+    }
+    cc->Outputs()
+        .Tag("TENSORS_GPU")
+        .Add(output_tensors.release(), cc->InputTimestamp());
+#endif
+  } else if (gpu_output_) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     // Output result tensors (GPU).
     auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
@@ -464,7 +533,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
           return ::mediapipe::OkStatus();
         }));
     cc->Outputs()
-        .Tag("TENSORS_GPU")
+        .Tag(kTensorsGpuTag)
         .Add(output_tensors.release(), cc->InputTimestamp());
 #elif defined(MEDIAPIPE_IOS)
     // Output result tensors (GPU).
@@ -488,7 +557,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     [convert_command endEncoding];
     [command_buffer commit];
     cc->Outputs()
-        .Tag("TENSORS_GPU")
+        .Tag(kTensorsGpuTag)
         .Add(output_tensors.release(), cc->InputTimestamp());
 #else
     RET_CHECK_FAIL() << "GPU processing not enabled.";
@@ -501,8 +570,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]);
       output_tensors->emplace_back(*tensor);
     }
-    cc->Outputs().Tag("TENSORS").Add(output_tensors.release(),
-                                     cc->InputTimestamp());
+    cc->Outputs()
+        .Tag(kTensorsTag)
+        .Add(output_tensors.release(), cc->InputTimestamp());
   }
 
   return ::mediapipe::OkStatus();
@@ -513,6 +583,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     if (gpu_inference_) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
       MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
+        interpreter_ = nullptr;
         delegate_ = nullptr;
         for (int i = 0; i < gpu_data_in_.size(); ++i) {
           gpu_data_in_[i].reset();
@@ -523,6 +594,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
         return ::mediapipe::OkStatus();
       }));
 #elif defined(MEDIAPIPE_IOS)
+      interpreter_ = nullptr;
       delegate_ = nullptr;
       for (int i = 0; i < gpu_data_in_.size(); ++i) {
         gpu_data_in_[i].reset();
@@ -532,6 +604,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       }
 #endif
     } else {
+      interpreter_ = nullptr;
       delegate_ = nullptr;
     }
   }
@@ -554,6 +627,20 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
                       .Tag("CUSTOM_OP_RESOLVER")
                       .Get<tflite::ops::builtin::BuiltinOpResolver>();
   }
+
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+  if (use_advanced_gpu_api_) {
+    tflite::gpu::InferenceOptions options;
+    options.priority1 = tflite::gpu::InferencePriority::MIN_LATENCY;
+    options.priority2 = tflite::gpu::InferencePriority::AUTO;
+    options.priority3 = tflite::gpu::InferencePriority::AUTO;
+    options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
+    tflite_gpu_runner_ =
+        std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
+    return tflite_gpu_runner_->InitializeWithModel(model);
+  }
+#endif
+
 #if defined(MEDIAPIPE_EDGE_TPU)
   interpreter_ =
       BuildEdgeTpuInterpreter(model, &op_resolver, edgetpu_context_.get());

mediapipe/calculators/tflite/tflite_inference_calculator.proto

@@ -42,7 +42,11 @@ message TfLiteInferenceCalculatorOptions {
     message TfLite {}
     // Delegate to run GPU inference depending on the device.
     // (Can use OpenGl, OpenCl, Metal depending on the device.)
-    message Gpu {}
+    message Gpu {
+      // Experimental, Android/Linux only. Use TFLite GPU delegate API2 for
+      // the NN inference.
+      optional bool use_advanced_gpu_api = 1 [default = false];
+    }
     // Android only.
     message Nnapi {}
     message Xnnpack {

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc

@@ -47,10 +47,11 @@
 #endif  // iOS
 
 namespace {
-
 constexpr int kNumInputTensorsWithAnchors = 3;
 constexpr int kNumCoordsPerBox = 4;
 
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
 }  // namespace
 
 namespace mediapipe {
@@ -99,7 +100,7 @@ void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
                                int num_boxes, float* raw_anchors) {
   CHECK_EQ(anchors.size(), num_boxes);
   int box = 0;
-  for (auto anchor : anchors) {
+  for (const auto& anchor : anchors) {
     raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center();
     raw_anchors[box * kNumCoordsPerBox + 1] = anchor.x_center();
     raw_anchors[box * kNumCoordsPerBox + 2] = anchor.h();
@@ -200,13 +201,13 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("TENSORS")) {
-    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+  if (cc->Inputs().HasTag(kTensorsTag)) {
+    cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
   }
 
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
-    cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
+    cc->Inputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -236,7 +237,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
     CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
     gpu_input_ = true;
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
@@ -258,8 +259,8 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::Process(
     CalculatorContext* cc) {
-  if ((!gpu_input_ && cc->Inputs().Tag("TENSORS").IsEmpty()) ||
-      (gpu_input_ && cc->Inputs().Tag("TENSORS_GPU").IsEmpty())) {
+  if ((!gpu_input_ && cc->Inputs().Tag(kTensorsTag).IsEmpty()) ||
+      (gpu_input_ && cc->Inputs().Tag(kTensorsGpuTag).IsEmpty())) {
     return ::mediapipe::OkStatus();
   }
 
@@ -284,7 +285,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::ProcessCPU(
     CalculatorContext* cc, std::vector<Detection>* output_detections) {
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+      cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
 
   if (input_tensors.size() == 2 ||
       input_tensors.size() == kNumInputTensorsWithAnchors) {
@@ -402,7 +403,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
     CalculatorContext* cc, std::vector<Detection>* output_detections) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+      cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
   RET_CHECK_GE(input_tensors.size(), 2);
 
   MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, &input_tensors, &cc,
@@ -466,7 +467,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 #elif defined(MEDIAPIPE_IOS)
 
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+      cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
   RET_CHECK_GE(input_tensors.size(), 2);
 
   // Copy inputs.

mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc

@@ -49,6 +49,16 @@ int NumGroups(const int size, const int group_size) {  // NOLINT
 float Clamp(float val, float min, float max) {
   return std::min(std::max(val, min), max);
 }
+
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
+constexpr char kSizeImageTag[] = "REFERENCE_IMAGE";
+constexpr char kSizeImageGpuTag[] = "REFERENCE_IMAGE_GPU";
+constexpr char kMaskTag[] = "MASK";
+constexpr char kMaskGpuTag[] = "MASK_GPU";
+constexpr char kPrevMaskTag[] = "PREV_MASK";
+constexpr char kPrevMaskGpuTag[] = "PREV_MASK_GPU";
+
 }  // namespace
 
 namespace mediapipe {
@@ -148,39 +158,39 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   bool use_gpu = false;
 
   // Inputs CPU.
-  if (cc->Inputs().HasTag("TENSORS")) {
-    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+  if (cc->Inputs().HasTag(kTensorsTag)) {
+    cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
   }
-  if (cc->Inputs().HasTag("PREV_MASK")) {
-    cc->Inputs().Tag("PREV_MASK").Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kPrevMaskTag)) {
+    cc->Inputs().Tag(kPrevMaskTag).Set<ImageFrame>();
   }
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE")) {
-    cc->Inputs().Tag("REFERENCE_IMAGE").Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kSizeImageTag)) {
+    cc->Inputs().Tag(kSizeImageTag).Set<ImageFrame>();
   }
 
   // Inputs GPU.
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
-    cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GlBuffer>>();
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
+    cc->Inputs().Tag(kTensorsGpuTag).Set<std::vector<GlBuffer>>();
     use_gpu |= true;
   }
-  if (cc->Inputs().HasTag("PREV_MASK_GPU")) {
-    cc->Inputs().Tag("PREV_MASK_GPU").Set<mediapipe::GpuBuffer>();
+  if (cc->Inputs().HasTag(kPrevMaskGpuTag)) {
+    cc->Inputs().Tag(kPrevMaskGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE_GPU")) {
-    cc->Inputs().Tag("REFERENCE_IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+  if (cc->Inputs().HasTag(kSizeImageGpuTag)) {
+    cc->Inputs().Tag(kSizeImageGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
   // Outputs.
-  if (cc->Outputs().HasTag("MASK")) {
-    cc->Outputs().Tag("MASK").Set<ImageFrame>();
+  if (cc->Outputs().HasTag(kMaskTag)) {
+    cc->Outputs().Tag(kMaskTag).Set<ImageFrame>();
   }
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-  if (cc->Outputs().HasTag("MASK_GPU")) {
-    cc->Outputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
+  if (cc->Outputs().HasTag(kMaskGpuTag)) {
+    cc->Outputs().Tag(kMaskGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -197,7 +207,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
     CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
     use_gpu_ = true;
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
@@ -255,23 +265,22 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 
 ::mediapipe::Status TfLiteTensorsToSegmentationCalculator::ProcessCpu(
     CalculatorContext* cc) {
-  if (cc->Inputs().Tag("TENSORS").IsEmpty()) {
+  if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
 
   // Get input streams.
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
-  const bool has_prev_mask = cc->Inputs().HasTag("PREV_MASK") &&
-                             !cc->Inputs().Tag("PREV_MASK").IsEmpty();
+      cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
+  const bool has_prev_mask = cc->Inputs().HasTag(kPrevMaskTag) &&
+                             !cc->Inputs().Tag(kPrevMaskTag).IsEmpty();
   const ImageFrame placeholder;
-  const auto& input_mask = has_prev_mask
-                               ? cc->Inputs().Tag("PREV_MASK").Get<ImageFrame>()
-                               : placeholder;
+  const auto& input_mask =
+      has_prev_mask ? cc->Inputs().Tag(kPrevMaskTag).Get<ImageFrame>()
+                    : placeholder;
   int output_width = tensor_width_, output_height = tensor_height_;
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE")) {
-    const auto& input_image =
-        cc->Inputs().Tag("REFERENCE_IMAGE").Get<ImageFrame>();
+  if (cc->Inputs().HasTag(kSizeImageTag)) {
+    const auto& input_image = cc->Inputs().Tag(kSizeImageTag).Get<ImageFrame>();
     output_width = input_image.Width();
     output_height = input_image.Height();
   }
@@ -353,7 +362,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
       ImageFormat::SRGBA, output_width, output_height);
   cv::Mat output_mat = formats::MatView(output_mask.get());
   large_mask_mat.copyTo(output_mat);
-  cc->Outputs().Tag("MASK").Add(output_mask.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
 }
@@ -364,23 +373,23 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 // 3. upsample small mask into output mask to be same size as input image
 ::mediapipe::Status TfLiteTensorsToSegmentationCalculator::ProcessGpu(
     CalculatorContext* cc) {
-  if (cc->Inputs().Tag("TENSORS_GPU").IsEmpty()) {
+  if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // Get input streams.
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GlBuffer>>();
-  const bool has_prev_mask = cc->Inputs().HasTag("PREV_MASK_GPU") &&
-                             !cc->Inputs().Tag("PREV_MASK_GPU").IsEmpty();
+      cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GlBuffer>>();
+  const bool has_prev_mask = cc->Inputs().HasTag(kPrevMaskGpuTag) &&
+                             !cc->Inputs().Tag(kPrevMaskGpuTag).IsEmpty();
   const auto& input_mask =
       has_prev_mask
-          ? cc->Inputs().Tag("PREV_MASK_GPU").Get<mediapipe::GpuBuffer>()
+          ? cc->Inputs().Tag(kPrevMaskGpuTag).Get<mediapipe::GpuBuffer>()
           : mediapipe::GpuBuffer();
   int output_width = tensor_width_, output_height = tensor_height_;
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kSizeImageGpuTag)) {
     const auto& input_image =
-        cc->Inputs().Tag("REFERENCE_IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+        cc->Inputs().Tag(kSizeImageGpuTag).Get<mediapipe::GpuBuffer>();
     output_width = input_image.width();
     output_height = input_image.height();
   }
@@ -441,7 +450,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   // Send out image as GPU packet.
   auto output_image = output_texture.GetFrame<mediapipe::GpuBuffer>();
   cc->Outputs()
-      .Tag("MASK_GPU")
+      .Tag(kMaskGpuTag)
       .Add(output_image.release(), cc->InputTimestamp());
 
   // Cleanup

mediapipe/calculators/util/detection_label_id_to_text_calculator.cc

@@ -71,16 +71,23 @@ REGISTER_CALCULATOR(DetectionLabelIdToTextCalculator);
   const auto& options =
       cc->Options<::mediapipe::DetectionLabelIdToTextCalculatorOptions>();
 
-  std::string string_path;
-  ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(options.label_map_path()));
-  std::string label_map_string;
-  MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
+  if (options.has_label_map_path()) {
+    std::string string_path;
+    ASSIGN_OR_RETURN(string_path,
+                     PathToResourceAsFile(options.label_map_path()));
+    std::string label_map_string;
+    MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
 
-  std::istringstream stream(label_map_string);
-  std::string line;
-  int i = 0;
-  while (std::getline(stream, line)) {
-    label_map_[i++] = line;
+    std::istringstream stream(label_map_string);
+    std::string line;
+    int i = 0;
+    while (std::getline(stream, line)) {
+      label_map_[i++] = line;
+    }
+  } else {
+    for (int i = 0; i < options.label_size(); ++i) {
+      label_map_[i] = options.label(i);
+    }
   }
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/util/detection_label_id_to_text_calculator.proto

@@ -25,4 +25,10 @@ message DetectionLabelIdToTextCalculatorOptions {
 
   // Path to a label map file for getting the actual name of detected classes.
   optional string label_map_path = 1;
+
+  // Alternative way to specify label map
+  // label: "label for id 0"
+  // label: "label for id 1"
+  // ...
+  repeated string label = 2;
 }

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -186,6 +186,7 @@ class LandmarksToRenderDataCalculator : public CalculatorBase {
 
  private:
   LandmarksToRenderDataCalculatorOptions options_;
+  std::vector<int> landmark_connections_;
 };
 REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
 
@@ -217,6 +218,14 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
   cc->SetOffset(TimestampDiff(0));
   options_ = cc->Options<LandmarksToRenderDataCalculatorOptions>();
 
+  // Parse landmarks connections to a vector.
+  RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
+      << "Number of entries in landmark connections must be a multiple of 2";
+
+  for (int i = 0; i < options_.landmark_connections_size(); ++i) {
+    landmark_connections_.push_back(options_.landmark_connections(i));
+  }
+
   return ::mediapipe::OkStatus();
 }
 
@@ -236,14 +245,6 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
     thickness *= render_scale;
   }
 
-  // Parse landmarks connections to a vector.
-  RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
-      << "Number of entries in landmark connections must be a multiple of 2";
-  std::vector<int> landmark_connections;
-  for (int i = 0; i < options_.landmark_connections_size(); i += 1) {
-    landmark_connections.push_back(options_.landmark_connections(i));
-  }
-
   if (cc->Inputs().HasTag(kLandmarksTag)) {
     const LandmarkList& landmarks =
         cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
@@ -252,6 +253,15 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
     }
     // Only change rendering if there are actually z values other than 0.
     visualize_depth &= ((z_max - z_min) > 1e-3);
+    if (visualize_depth) {
+      AddConnectionsWithDepth<LandmarkList, Landmark>(
+          landmarks, landmark_connections_, thickness, /*normalized=*/false,
+          z_min, z_max, render_data.get());
+    } else {
+      AddConnections<LandmarkList, Landmark>(
+          landmarks, landmark_connections_, options_.connection_color(),
+          thickness, /*normalized=*/false, render_data.get());
+    }
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const Landmark& landmark = landmarks.landmark(i);
       auto* landmark_data_render = AddPointRenderData(
@@ -265,15 +275,6 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
       landmark_data->set_x(landmark.x());
       landmark_data->set_y(landmark.y());
     }
-    if (visualize_depth) {
-      AddConnectionsWithDepth<LandmarkList, Landmark>(
-          landmarks, landmark_connections, thickness, /*normalized=*/false,
-          z_min, z_max, render_data.get());
-    } else {
-      AddConnections<LandmarkList, Landmark>(
-          landmarks, landmark_connections, options_.connection_color(),
-          thickness, /*normalized=*/false, render_data.get());
-    }
   }
 
   if (cc->Inputs().HasTag(kNormLandmarksTag)) {
@@ -285,6 +286,15 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
     }
     // Only change rendering if there are actually z values other than 0.
     visualize_depth &= ((z_max - z_min) > 1e-3);
+    if (visualize_depth) {
+      AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
+          landmarks, landmark_connections_, thickness, /*normalized=*/true,
+          z_min, z_max, render_data.get());
+    } else {
+      AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
+          landmarks, landmark_connections_, options_.connection_color(),
+          thickness, /*normalized=*/true, render_data.get());
+    }
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const NormalizedLandmark& landmark = landmarks.landmark(i);
       auto* landmark_data_render = AddPointRenderData(
@@ -298,15 +308,6 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
       landmark_data->set_x(landmark.x());
       landmark_data->set_y(landmark.y());
     }
-    if (visualize_depth) {
-      AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
-          landmarks, landmark_connections, thickness, /*normalized=*/true,
-          z_min, z_max, render_data.get());
-    } else {
-      AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
-          landmarks, landmark_connections, options_.connection_color(),
-          thickness, /*normalized=*/true, render_data.get());
-    }
   }
 
   cc->Outputs()

mediapipe/docs/android_archive_library.md

@@ -32,13 +32,14 @@ project.
 2.  Run the Bazel build command to generate the AAR.
 
     ```bash
-    bazel build -c opt --fat_apk_cpu=arm64-v8a,armeabi-v7a //path/to/the/aar/build/file:aar_name
+    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+        //path/to/the/aar/build/file:aar_name
     ```
 
     For the face detection AAR target we made in the step 1, run:
 
     ```bash
-    bazel build -c opt --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
         //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar
 
     # It should print:

mediapipe/docs/examples.md

@@ -73,6 +73,18 @@ can be easily adapted to run on CPU v.s. GPU.
 *   [Android](./face_detection_mobile_cpu.md)
 *   [iOS](./face_detection_mobile_cpu.md)
 
+### Face Mesh with GPU
+
+[Face Mesh with GPU](./face_mesh_mobile_gpu.md) illustrates how to run the
+MediaPipe Face Mesh pipeline to perform 3D face landmark estimation in real-time
+on mobile devices, utilizing GPU acceleration. The pipeline is based on
+["Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs"](https://arxiv.org/abs/1907.06724),
+and details of the underlying ML models are described in the
+[model card](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view).
+
+*   [Android](./face_mesh_mobile_gpu.md)
+*   [iOS](./face_mesh_mobile_gpu.md)
+
 ### Hand Detection with GPU
 
 [Hand Detection with GPU](./hand_detection_mobile_gpu.md) illustrates how to use
@@ -84,7 +96,7 @@ MediaPipe with a TFLite model for hand detection in a GPU-accelerated pipeline.
 ### Hand Tracking with GPU
 
 [Hand Tracking with GPU](./hand_tracking_mobile_gpu.md) illustrates how to use
-MediaPipe with a TFLite model for hand tracking in a GPU-accelerated pipeline.
+MediaPipe with TFLite models for hand tracking in a GPU-accelerated pipeline.
 
 *   [Android](./hand_tracking_mobile_gpu.md)
 *   [iOS](./hand_tracking_mobile_gpu.md)
@@ -92,7 +104,7 @@ MediaPipe with a TFLite model for hand tracking in a GPU-accelerated pipeline.
 ### Multi-Hand Tracking with GPU
 
 [Multi-Hand Tracking with GPU](./multi_hand_tracking_mobile_gpu.md) illustrates
-how to use MediaPipe with a TFLite model for multi-hand tracking in a
+how to use MediaPipe with TFLite models for multi-hand tracking in a
 GPU-accelerated pipeline.
 
 *   [Android](./multi_hand_tracking_mobile_gpu.md)
@@ -109,6 +121,14 @@ and model details are described in the
 
 *   [Android](./hair_segmentation_mobile_gpu.md)
 
+### Template Matching using KNIFT with CPU
+
+[Template Matching using KNIFT on Mobile](./template_matching_mobile_cpu.md)
+shows how to use MediaPipe with TFLite model for template matching using Knift
+on mobile using CPU.
+
+*   [Android](./template_matching_mobile_cpu.md)
+
 ## Desktop
 
 ### Hello World for C++
@@ -150,11 +170,19 @@ GPU with live video from a webcam.
 *   [Desktop GPU](./face_detection_desktop.md)
 *   [Desktop CPU](./face_detection_desktop.md)
 
+### Face Mesh on Desktop with Webcam
+
+[Face Mesh on Desktop with Webcam](./face_mesh_desktop.md) shows how to run the
+MediaPipe Face Mesh pipeline to perform 3D face landmark estimation in real-time
+on desktop with webcam input.
+
+*   [Desktop GPU](./face_mesh_desktop.md)
+*   [Desktop CPU](./face_mesh_desktop.md)
 
 ### Hand Tracking on Desktop with Webcam
 
 [Hand Tracking on Desktop with Webcam](./hand_tracking_desktop.md) shows how to
-use MediaPipe with a TFLite model for hand tracking on desktop using CPU or GPU
+use MediaPipe with TFLite models for hand tracking on desktop using CPU or GPU
 with live video from a webcam.
 
 *   [Desktop GPU](./hand_tracking_desktop.md)
@@ -163,8 +191,8 @@ with live video from a webcam.
 ### Multi-Hand Tracking on Desktop with Webcam
 
 [Multi-Hand Tracking on Desktop with Webcam](./multi_hand_tracking_desktop.md)
-shows how to use MediaPipe with a TFLite model for multi-hand tracking on
-desktop using CPU or GPU with live video from a webcam.
+shows how to use MediaPipe with TFLite models for multi-hand tracking on desktop
+using CPU or GPU with live video from a webcam.
 
 *   [Desktop GPU](./multi_hand_tracking_desktop.md)
 *   [Desktop CPU](./multi_hand_tracking_desktop.md)
@@ -177,7 +205,7 @@ GPU with live video from a webcam.
 
 *   [Desktop GPU](./hair_segmentation_desktop.md)
 
-## Google Coral (machine learning acceleration with Google EdgeTPU)
+## Google Coral (ML acceleration with Google EdgeTPU)
 
 Below are code samples on how to run MediaPipe on Google Coral Dev Board.
 

mediapipe/docs/face_detection_mobile_cpu.md

@@ -4,6 +4,8 @@ This doc focuses on the
 [example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt)
 that performs face detection with TensorFlow Lite on CPU.
 
+![face_detection_android_gpu_gif](images/mobile/face_detection_android_gpu.gif)
+
 ## Android
 
 [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu)

mediapipe/docs/face_mesh_desktop.md

@@ -0,0 +1,58 @@
+## Face Mesh on Desktop with Webcam
+
+This doc focuses on running the **MediaPipe Face Mesh** pipeline to perform 3D
+face landmark estimation in real-time on desktop with webcam input. The pipeline
+internally incorporates TensorFlow Lite models. To know more about the models,
+please refer to the model
+[README file](https://github.com/google/mediapipe/tree/master/mediapipe/models/README.md#face-mesh).
+Moreover, if you are interested in running the same pipeline on Android/iOS,
+please see [Face Mesh on Android/iOS](face_mesh_mobile_gpu.md).
+
+-   [Face Mesh on Desktop with Webcam (CPU)](#face-mesh-on-desktop-with-webcam-cpu)
+
+-   [Face Mesh on Desktop with Webcam (GPU)](#face-mesh-on-desktop-with-webcam-gpu)
+
+Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
+see
+[step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
+
+Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+
+### Face Mesh on Desktop with Webcam (CPU)
+
+To build and run Face Mesh on desktop with webcam (CPU), run:
+
+```bash
+$ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+    mediapipe/examples/desktop/face_mesh:face_mesh_cpu
+
+# It should print:
+# Target //mediapipe/examples/desktop/face_mesh:face_mesh_cpu up-to-date:
+#  bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_cpu
+
+# This will open up your webcam as long as it is connected. Errors are likely
+# due to your webcam being not accessible.
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_cpu \
+    --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt
+```
+
+### Face Mesh on Desktop with Webcam (GPU)
+
+Note: please first [check that your GPU is supported](gpu.md#desktop-gpu-linux).
+
+To build and run Face Mesh on desktop with webcam (GPU), run:
+
+```bash
+# This works only for linux currently
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
+    mediapipe/examples/desktop/face_mesh:face_mesh_gpu
+
+# It should print:
+# Target //mediapipe/examples/desktop/face_mesh:face_mesh_gpu up-to-date:
+#  bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_gpu
+
+# This will open up your webcam as long as it is connected. Errors are likely
+# due to your webcam being not accessible, or GPU drivers not setup properly.
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_gpu \
+    --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt
+```

mediapipe/docs/face_mesh_mobile_gpu.md

@@ -0,0 +1,90 @@
+# Face Mesh (GPU)
+
+This example focuses on running the **MediaPipe Face Mesh** pipeline on mobile
+devices to perform 3D face landmark estimation in real-time, utilizing GPU
+acceleration. The pipeline internally incorporates TensorFlow Lite models. To
+know more about the models, please refer to the model
+[README file](https://github.com/google/mediapipe/tree/master/mediapipe/models/README.md#face-mesh).
+The pipeline is related to the
+[face detection example](./face_detection_mobile_gpu.md) as it internally
+utilizes face detection and performs landmark estimation only within the
+detected region.
+
+![face_mesh_android_gpu.gif](images/mobile/face_mesh_android_gpu.gif)
+
+**MediaPipe Face Mesh** generates 468 3D face landmarks in real-time on mobile
+devices. In the visualization above, the red dots represent the landmarks, and
+the green lines connecting landmarks illustrate the contours around the eyes,
+eyebrows, lips and the entire face.
+
+## Android
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu)
+
+A prebuilt arm64 APK can be
+[downloaded here](https://drive.google.com/open?id=1pUmd7CXCL_onYMbsZo5p91cH0oNnR4gi).
+
+To build the app yourself, run:
+
+```bash
+bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu
+```
+
+Once the app is built, install it on Android device with:
+
+```bash
+adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/facemeshgpu.apk
+```
+
+## iOS
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facemeshgpu).
+
+See the general [instructions](./mediapipe_ios_setup.md) for building iOS
+examples and generating an Xcode project. This will be the FaceMeshGpuApp
+target.
+
+To build on the command line:
+
+```bash
+bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp
+```
+
+## Graph
+
+The face mesh [main graph](#main-graph) utilizes a
+[face landmark subgraph](#face-landmark-subgraph) from the
+[face landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark),
+and renders using a dedicated [face renderer subgraph](#face-renderer-subgraph).
+
+The subgraphs show up in the main graph visualization as nodes colored in
+purple, and the subgraph itself can also be visualized just like a regular
+graph. For more information on how to visualize a graph that includes subgraphs,
+see the Visualizing Subgraphs section in the
+[visualizer documentation](./visualizer.md).
+
+### Main Graph
+
+![face_mesh_mobile_graph](images/mobile/face_mesh_mobile.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
+
+### Face Landmark Subgraph
+
+The
+[face landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark)
+contains several subgraphs that can be used to detect and track face landmarks.
+In particular, in this example the
+[FaceLandmarkFrontGPU](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
+subgraph, suitable for images from front-facing cameras (i.e., selfie images)
+and utilizing GPU acceleration, is selected.
+
+![face_landmark_front_gpu_subgraph](images/mobile/face_landmark_front_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
+
+### Face Renderer Subgraph
+
+![face_renderer_gpu_subgraph](images/mobile/face_renderer_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/subgraphs/face_renderer_gpu.pbtxt)

mediapipe/docs/object_detection_desktop.md

@@ -12,7 +12,7 @@ We show the object detection demo with both TensorFlow model and TensorFlow Lite
 
 -   [TensorFlow Object Detection Demo](#tensorflow-object-detection-demo)
 -   [TensorFlow Lite Object Detection Demo](#tensorflow-lite-object-detection-demo)
--   [TensorFlow Lite Object Detection Demo with Webcam (CPU)](#tensorflow-lite-object-detection-demo)
+-   [TensorFlow Lite Object Detection Demo with Webcam (CPU)](#tensorflow-lite-object-detection-demo-with-webcam-cpu)
 
 Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
 

mediapipe/docs/template_matching_desktop_cpu.md

@@ -0,0 +1,31 @@
+# Template Matching using KNIFT on Desktop
+
+This doc focuses on the
+[example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/template_matching/template_matching_desktop.pbtxt)
+that performs template matching with KNIFT (Keypoint Neural Invariant Feature
+Transform) on desktop CPU.
+
+If you are interested in more detail about KNIFT or running the example on
+mobile, please see
+[Template Matching using KNIFT on Mobile (CPU)](template_matching_mobile_cpu.md).
+
+To build the desktop app, run:
+
+```bash
+$ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+    mediapipe/examples/desktop/template_matching:template_matching_tflite
+```
+
+To run the desktop app, please specify a template index file
+([example](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_index.pb)) and a
+video to be matched. For how to build your own index file, please see
+[here](template_matching_mobile_cpu.md#build-index-file).
+
+```bash
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/template_matching/template_matching_tflite \
+    --calculator_graph_config_file=mediapipe/graphs/template_matching/template_matching_desktop.pbtxt --input_side_packets="input_video_path=<input video path>,output_video_path=<output video path>"
+```
+
+## Graph
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/template_matching/template_matching_desktop.pbtxt)

mediapipe/docs/template_matching_mobile_cpu.md

@@ -0,0 +1,94 @@
+# Template Matching using KNIFT on Mobile (CPU)
+
+This doc focuses on the
+[example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/template_matching/template_matching_mobile_cpu.pbtxt)
+that performs template matching with KNIFT (Keypoint Neural Invariant Feature
+Transform) on mobile CPU.
+
+![template_matching_mobile_cpu.gif](images/mobile/template_matching_android_cpu.gif)
+
+In the visualization above, the green dots represent detected keypoints on each
+frame and the red box represents the targets matched by templates using KNIFT
+features (see also [model card](https://mediapipe.page.link/knift-mc)). For more
+information, please see
+[Google Developers Blog](https://mediapipe.page.link/knift-blog).
+
+## Build Index Files
+
+In MediaPipe, we've already provided a file in
+[knift_index.pb](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_index.pb),
+pre-computed from the 3 template images (of USD bills) shown below. If you'd
+like to use your own template images, please follow the steps below, or
+otherwise you can jump directly to [Android](#android).
+
+![template_matching_mobile_template.jpg](images/mobile/template_matching_mobile_template.jpg)
+
+### Step 1:
+
+Put all template images in a single directory.
+
+### Step 2:
+
+To build the index file for all templates in the directory, run:
+
+```bash
+$ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+    mediapipe/examples/desktop/template_matching:template_matching_tflite
+$ bazel-bin/mediapipe/examples/desktop/template_matching/template_matching_tflite \
+    --calculator_graph_config_file=mediapipe/graphs/template_matching/index_building.pbtxt \
+    --input_side_packets="file_directory=<template image directory>,file_suffix='png',output_index_filename=<output index filename>"
+```
+
+The output index file includes the extracted KNIFT features.
+
+### Step 3:
+
+Replace
+[mediapipe/models/knift_index.pb](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_index.pb)
+with the index file you generated, and update
+[mediapipe/models/knift_labelmap.txt](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_labelmap.txt)
+with your own template names.
+
+## Android
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu)
+
+A prebuilt arm64 APK can be
+[downloaded here](https://drive.google.com/open?id=1tSWRfes9rAM4NrzmJBplguNQQvaeBZSa).
+
+To build and install the app yourself, run:
+
+Note: MediaPipe uses OpenCV 3 by default. However, because of
+[issues](https://github.com/opencv/opencv/issues/11488) between NDK 17+ and
+OpenCV 3 when using
+[knnMatch](https://docs.opencv.org/3.4/db/d39/classcv_1_1DescriptorMatcher.html#a378f35c9b1a5dfa4022839a45cdf0e89),
+please use the following commands to temporarily switch to OpenCV 4 for the
+template matching exmaple on Android, and switch back to OpenCV 3 afterwards.
+
+```bash
+# Switch to OpenCV 4
+sed -i -e 's:3.4.3/opencv-3.4.3:4.0.1/opencv-4.0.1:g' WORKSPACE
+sed -i -e 's:libopencv_java3:libopencv_java4:g' third_party/opencv_android.BUILD
+
+# Build and install app
+bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu:templatematchingcpu
+adb install -r bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/templatematchingcpu.apk
+
+# Switch back to OpenCV 3
+sed -i -e 's:4.0.1/opencv-4.0.1:3.4.3/opencv-3.4.3:g' WORKSPACE
+sed -i -e 's:libopencv_java4:libopencv_java3:g' third_party/opencv_android.BUILD
+```
+
+## Use XNNPACK Delegate
+
+The example uses XNNPACK delegate by default. Users can change the
+[option in TfLiteInferenceCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/calculators/tflite/tflite_inference_calculator.proto)
+to use default TF Lite inference.
+
+## Graph
+
+### Main Graph
+
+![template_matching_mobile_graph](images/mobile/template_matching_mobile_graph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/template_matching/template_matching_mobile_cpu.pbtxt)

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/AndroidManifest.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.apps.facemeshgpu">
+
+  <uses-sdk
+      android:minSdkVersion="21"
+      android:targetSdkVersion="27" />
+
+  <!-- For using the camera -->
+  <uses-permission android:name="android.permission.CAMERA" />
+  <uses-feature android:name="android.hardware.camera" />
+  <uses-feature android:name="android.hardware.camera.autofocus" />
+  <!-- For MediaPipe -->
+  <uses-feature android:glEsVersion="0x00020000" android:required="true" />
+
+
+  <application
+      android:allowBackup="true"
+      android:label="@string/app_name"
+      android:supportsRtl="true"
+      android:theme="@style/AppTheme">
+      <activity
+          android:name=".MainActivity"
+          android:exported="true"
+          android:screenOrientation="portrait">
+          <intent-filter>
+              <action android:name="android.intent.action.MAIN" />
+              <category android:name="android.intent.category.LAUNCHER" />
+          </intent-filter>
+      </activity>
+  </application>
+
+</manifest>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD

@@ -0,0 +1,82 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "libmediapipe_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/graphs/face_mesh:mobile_calculators",
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+    ],
+)
+
+cc_library(
+    name = "mediapipe_jni_lib",
+    srcs = [":libmediapipe_jni.so"],
+    alwayslink = 1,
+)
+
+# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be
+# easily incorporated into the app via, for example,
+# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb".
+genrule(
+    name = "binary_graph",
+    srcs = ["//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu_binary_graph"],
+    outs = ["facemeshgpu.binarypb"],
+    cmd = "cp $< $@",
+)
+
+android_library(
+    name = "mediapipe_lib",
+    srcs = glob(["*.java"]),
+    assets = [
+        ":binary_graph",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+    ],
+    assets_dir = "",
+    manifest = "AndroidManifest.xml",
+    resource_files = glob(["res/**"]),
+    deps = [
+        ":mediapipe_jni_lib",
+        "//mediapipe/framework/formats:landmark_java_proto_lite",
+        "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
+        "//mediapipe/java/com/google/mediapipe/components:android_components",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/java/com/google/mediapipe/glutil",
+        "//third_party:androidx_appcompat",
+        "//third_party:androidx_constraint_layout",
+        "//third_party:androidx_legacy_support_v4",
+        "//third_party:androidx_recyclerview",
+        "//third_party:opencv",
+        "@maven//:androidx_concurrent_concurrent_futures",
+        "@maven//:androidx_lifecycle_lifecycle_common",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
+android_binary(
+    name = "facemeshgpu",
+    manifest = "AndroidManifest.xml",
+    manifest_values = {"applicationId": "com.google.mediapipe.apps.facemeshgpu"},
+    multidex = "native",
+    deps = [
+        ":mediapipe_lib",
+    ],
+)

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/MainActivity.java

@@ -0,0 +1,232 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.apps.facemeshgpu;
+
+import android.graphics.SurfaceTexture;
+import android.os.Bundle;
+import androidx.appcompat.app.AppCompatActivity;
+import android.util.Log;
+import android.util.Size;
+import android.view.SurfaceHolder;
+import android.view.SurfaceView;
+import android.view.View;
+import android.view.ViewGroup;
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
+import com.google.mediapipe.components.CameraHelper;
+import com.google.mediapipe.components.CameraXPreviewHelper;
+import com.google.mediapipe.components.ExternalTextureConverter;
+import com.google.mediapipe.components.FrameProcessor;
+import com.google.mediapipe.components.PermissionHelper;
+import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.framework.AndroidPacketCreator;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.framework.PacketGetter;
+import com.google.mediapipe.glutil.EglManager;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Main activity of MediaPipe example apps. */
+public class MainActivity extends AppCompatActivity {
+  private static final String TAG = "MainActivity";
+
+  private static final String BINARY_GRAPH_NAME = "facemeshgpu.binarypb";
+  private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
+  private static final String INPUT_NUM_FACES_SIDE_PACKET_NAME = "num_faces";
+  private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final String OUTPUT_LANDMARKS_STREAM_NAME = "multi_face_landmarks";
+  private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.FRONT;
+
+  // Max number of faces to detect/process.
+  private static final int NUM_FACES = 1;
+
+  // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
+  // processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
+  // This is needed because OpenGL represents images assuming the image origin is at the bottom-left
+  // corner, whereas MediaPipe in general assumes the image origin is at top-left.
+  private static final boolean FLIP_FRAMES_VERTICALLY = true;
+
+  static {
+    // Load all native libraries needed by the app.
+    System.loadLibrary("mediapipe_jni");
+    System.loadLibrary("opencv_java3");
+  }
+
+  // {@link SurfaceTexture} where the camera-preview frames can be accessed.
+  private SurfaceTexture previewFrameTexture;
+  // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
+  private SurfaceView previewDisplayView;
+
+  // Creates and manages an {@link EGLContext}.
+  private EglManager eglManager;
+  // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
+  // frames onto a {@link Surface}.
+  private FrameProcessor processor;
+  // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
+  // consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
+  private ExternalTextureConverter converter;
+
+  // Handles camera access via the {@link CameraX} Jetpack support library.
+  private CameraXPreviewHelper cameraHelper;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_main);
+
+    previewDisplayView = new SurfaceView(this);
+    setupPreviewDisplayView();
+
+    // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
+    // binary graphs.
+    AndroidAssetUtil.initializeNativeAssetManager(this);
+
+    eglManager = new EglManager(null);
+    processor =
+        new FrameProcessor(
+            this,
+            eglManager.getNativeContext(),
+            BINARY_GRAPH_NAME,
+            INPUT_VIDEO_STREAM_NAME,
+            OUTPUT_VIDEO_STREAM_NAME);
+    processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
+
+    AndroidPacketCreator packetCreator = processor.getPacketCreator();
+    Map<String, Packet> inputSidePackets = new HashMap<>();
+    inputSidePackets.put(INPUT_NUM_FACES_SIDE_PACKET_NAME, packetCreator.createInt32(NUM_FACES));
+    processor.setInputSidePackets(inputSidePackets);
+
+    processor.addPacketCallback(
+        OUTPUT_LANDMARKS_STREAM_NAME,
+        (packet) -> {
+          Log.d(TAG, "Received multi face landmarks packet.");
+          List<NormalizedLandmarkList> multiFaceLandmarks =
+              PacketGetter.getProtoVector(packet, NormalizedLandmarkList.parser());
+          Log.d(
+              TAG,
+              "[TS:"
+                  + packet.getTimestamp()
+                  + "] "
+                  + getMultiFaceLandmarksDebugString(multiFaceLandmarks));
+        });
+
+    PermissionHelper.checkAndRequestCameraPermissions(this);
+  }
+
+  @Override
+  protected void onResume() {
+    super.onResume();
+    converter = new ExternalTextureConverter(eglManager.getContext());
+    converter.setFlipY(FLIP_FRAMES_VERTICALLY);
+    converter.setConsumer(processor);
+    if (PermissionHelper.cameraPermissionsGranted(this)) {
+      startCamera();
+    }
+  }
+
+  @Override
+  protected void onPause() {
+    super.onPause();
+    converter.close();
+  }
+
+  @Override
+  public void onRequestPermissionsResult(
+      int requestCode, String[] permissions, int[] grantResults) {
+    super.onRequestPermissionsResult(requestCode, permissions, grantResults);
+    PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
+  }
+
+  private void setupPreviewDisplayView() {
+    previewDisplayView.setVisibility(View.GONE);
+    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
+    viewGroup.addView(previewDisplayView);
+
+    previewDisplayView
+        .getHolder()
+        .addCallback(
+            new SurfaceHolder.Callback() {
+              @Override
+              public void surfaceCreated(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
+              }
+
+              @Override
+              public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+                // (Re-)Compute the ideal size of the camera-preview display (the area that the
+                // camera-preview frames get rendered onto, potentially with scaling and rotation)
+                // based on the size of the SurfaceView that contains the display.
+                Size viewSize = new Size(width, height);
+                Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
+
+                // Connect the converter to the camera-preview frames as its input (via
+                // previewFrameTexture), and configure the output width and height as the computed
+                // display size.
+                converter.setSurfaceTextureAndAttachToGLContext(
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
+              }
+
+              @Override
+              public void surfaceDestroyed(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(null);
+              }
+            });
+  }
+
+  private void startCamera() {
+    cameraHelper = new CameraXPreviewHelper();
+    cameraHelper.setOnCameraStartedListener(
+        surfaceTexture -> {
+          previewFrameTexture = surfaceTexture;
+          // Make the display view visible to start showing the preview. This triggers the
+          // SurfaceHolder.Callback added to (the holder of) previewDisplayView.
+          previewDisplayView.setVisibility(View.VISIBLE);
+        });
+    cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
+  }
+
+  private static String getMultiFaceLandmarksDebugString(
+      List<NormalizedLandmarkList> multiFaceLandmarks) {
+    if (multiFaceLandmarks.isEmpty()) {
+      return "No face landmarks";
+    }
+    String multiFaceLandmarksStr = "Number of faces detected: " + multiFaceLandmarks.size() + "\n";
+    int faceIndex = 0;
+    for (NormalizedLandmarkList landmarks : multiFaceLandmarks) {
+      multiFaceLandmarksStr +=
+          "\t#Face landmarks for face[" + faceIndex + "]: " + landmarks.getLandmarkCount() + "\n";
+      int landmarkIndex = 0;
+      for (NormalizedLandmark landmark : landmarks.getLandmarkList()) {
+        multiFaceLandmarksStr +=
+            "\t\tLandmark ["
+                + landmarkIndex
+                + "]: ("
+                + landmark.getX()
+                + ", "
+                + landmark.getY()
+                + ", "
+                + landmark.getZ()
+                + ")\n";
+        ++landmarkIndex;
+      }
+      ++faceIndex;
+    }
+    return multiFaceLandmarksStr;
+  }
+}

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/res/layout/activity_main.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <FrameLayout
+        android:id="@+id/preview_display_layout"
+        android:layout_width="fill_parent"
+        android:layout_height="fill_parent"
+        android:layout_weight="1">
+        <TextView
+            android:id="@+id/no_camera_access_view"
+            android:layout_height="fill_parent"
+            android:layout_width="fill_parent"
+            android:gravity="center"
+            android:text="@string/no_camera_access" />
+    </FrameLayout>
+</androidx.constraintlayout.widget.ConstraintLayout>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/res/values/colors.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <color name="colorPrimary">#008577</color>
+    <color name="colorPrimaryDark">#00574B</color>
+    <color name="colorAccent">#D81B60</color>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/res/values/strings.xml

@@ -0,0 +1,4 @@
+<resources>
+    <string name="app_name" translatable="false">Face Mesh GPU</string>
+    <string name="no_camera_access" translatable="false">Please grant camera permissions.</string>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/res/values/styles.xml

@@ -0,0 +1,11 @@
+<resources>
+
+    <!-- Base application theme. -->
+    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+        <!-- Customize your theme here. -->
+        <item name="colorPrimary">@color/colorPrimary</item>
+        <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
+        <item name="colorAccent">@color/colorAccent</item>
+    </style>
+
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/AndroidManifest.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.apps.templatematchingcpu">
+
+  <uses-sdk
+      android:minSdkVersion="21"
+      android:targetSdkVersion="27" />
+
+  <!-- For using the camera -->
+  <uses-permission android:name="android.permission.CAMERA" />
+  <uses-feature android:name="android.hardware.camera" />
+  <uses-feature android:name="android.hardware.camera.autofocus" />
+  <!-- For MediaPipe -->
+  <uses-feature android:glEsVersion="0x00020000" android:required="true" />
+
+
+  <application
+      android:allowBackup="true"
+      android:label="@string/app_name"
+      android:supportsRtl="true"
+      android:theme="@style/AppTheme">
+      <activity
+          android:name=".MainActivity"
+          android:exported="true"
+          android:screenOrientation="portrait">
+          <intent-filter>
+              <action android:name="android.intent.action.MAIN" />
+              <category android:name="android.intent.category.LAUNCHER" />
+          </intent-filter>
+      </activity>
+  </application>
+
+</manifest>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/BUILD

@@ -0,0 +1,82 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "libmediapipe_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/graphs/template_matching:mobile_calculators",
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+    ],
+)
+
+cc_library(
+    name = "mediapipe_jni_lib",
+    srcs = [":libmediapipe_jni.so"],
+    alwayslink = 1,
+)
+
+# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be
+# easily incorporated into the app via, for example,
+# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb".
+genrule(
+    name = "binary_graph",
+    srcs = ["//mediapipe/graphs/template_matching:mobile_cpu_binary_graph"],
+    outs = ["templatematching.binarypb"],
+    cmd = "cp $< $@",
+)
+
+android_library(
+    name = "mediapipe_lib",
+    srcs = glob(["*.java"]),
+    assets = [
+        ":binary_graph",
+        "//mediapipe/models:knift_index.pb",
+        "//mediapipe/models:knift_float.tflite",
+        "//mediapipe/models:knift_labelmap.txt",
+    ],
+    assets_dir = "",
+    manifest = "AndroidManifest.xml",
+    resource_files = glob(["res/**"]),
+    deps = [
+        ":mediapipe_jni_lib",
+        "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
+        "//mediapipe/java/com/google/mediapipe/components:android_components",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/java/com/google/mediapipe/glutil",
+        "//third_party:androidx_appcompat",
+        "//third_party:androidx_constraint_layout",
+        "//third_party:androidx_legacy_support_v4",
+        "//third_party:androidx_recyclerview",
+        "//third_party:opencv",
+        "@maven//:androidx_concurrent_concurrent_futures",
+        "@maven//:androidx_lifecycle_lifecycle_common",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
+android_binary(
+    name = "templatematchingcpu",
+    manifest = "AndroidManifest.xml",
+    manifest_values = {"applicationId": "com.google.mediapipe.apps.templatematchingcpu"},
+    multidex = "native",
+    deps = [
+        ":mediapipe_lib",
+    ],
+)

mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/MainActivity.java

@@ -0,0 +1,170 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.apps.templatematchingcpu;
+
+import android.graphics.SurfaceTexture;
+import android.os.Bundle;
+import androidx.appcompat.app.AppCompatActivity;
+import android.util.Size;
+import android.view.SurfaceHolder;
+import android.view.SurfaceView;
+import android.view.View;
+import android.view.ViewGroup;
+import com.google.mediapipe.components.CameraHelper;
+import com.google.mediapipe.components.CameraXPreviewHelper;
+import com.google.mediapipe.components.ExternalTextureConverter;
+import com.google.mediapipe.components.FrameProcessor;
+import com.google.mediapipe.components.PermissionHelper;
+import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.glutil.EglManager;
+
+/** Main activity of MediaPipe example apps. */
+public class MainActivity extends AppCompatActivity {
+  private static final String TAG = "MainActivity";
+
+  private static final String BINARY_GRAPH_NAME = "templatematching.binarypb";
+  private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
+  private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
+
+  // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
+  // processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
+  // This is needed because OpenGL represents images assuming the image origin is at the bottom-left
+  // corner, whereas MediaPipe in general assumes the image origin is at top-left.
+  private static final boolean FLIP_FRAMES_VERTICALLY = true;
+
+  static {
+    // Load all native libraries needed by the app.
+    System.loadLibrary("mediapipe_jni");
+    System.loadLibrary("opencv_java4");
+  }
+
+  // {@link SurfaceTexture} where the camera-preview frames can be accessed.
+  private SurfaceTexture previewFrameTexture;
+  // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
+  private SurfaceView previewDisplayView;
+
+  // Creates and manages an {@link EGLContext}.
+  private EglManager eglManager;
+  // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
+  // frames onto a {@link Surface}.
+  private FrameProcessor processor;
+  // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
+  // consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
+  private ExternalTextureConverter converter;
+
+  // Handles camera access via the {@link CameraX} Jetpack support library.
+  private CameraXPreviewHelper cameraHelper;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_main);
+
+    previewDisplayView = new SurfaceView(this);
+    setupPreviewDisplayView();
+
+    // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
+    // binary graphs.
+    AndroidAssetUtil.initializeNativeAssetManager(this);
+
+    eglManager = new EglManager(null);
+    processor =
+        new FrameProcessor(
+            this,
+            eglManager.getNativeContext(),
+            BINARY_GRAPH_NAME,
+            INPUT_VIDEO_STREAM_NAME,
+            OUTPUT_VIDEO_STREAM_NAME);
+    processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
+
+    PermissionHelper.checkAndRequestCameraPermissions(this);
+  }
+
+  @Override
+  protected void onResume() {
+    super.onResume();
+    converter = new ExternalTextureConverter(eglManager.getContext());
+    converter.setFlipY(FLIP_FRAMES_VERTICALLY);
+    converter.setConsumer(processor);
+    if (PermissionHelper.cameraPermissionsGranted(this)) {
+      startCamera();
+    }
+  }
+
+  @Override
+  protected void onPause() {
+    super.onPause();
+    converter.close();
+  }
+
+  @Override
+  public void onRequestPermissionsResult(
+      int requestCode, String[] permissions, int[] grantResults) {
+    super.onRequestPermissionsResult(requestCode, permissions, grantResults);
+    PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
+  }
+
+  private void setupPreviewDisplayView() {
+    previewDisplayView.setVisibility(View.GONE);
+    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
+    viewGroup.addView(previewDisplayView);
+
+    previewDisplayView
+        .getHolder()
+        .addCallback(
+            new SurfaceHolder.Callback() {
+              @Override
+              public void surfaceCreated(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
+              }
+
+              @Override
+              public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+                // (Re-)Compute the ideal size of the camera-preview display (the area that the
+                // camera-preview frames get rendered onto, potentially with scaling and rotation)
+                // based on the size of the SurfaceView that contains the display.
+                Size viewSize = new Size(width, height);
+                Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
+
+                // Connect the converter to the camera-preview frames as its input (via
+                // previewFrameTexture), and configure the output width and height as the computed
+                // display size.
+                converter.setSurfaceTextureAndAttachToGLContext(
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
+              }
+
+              @Override
+              public void surfaceDestroyed(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(null);
+              }
+            });
+  }
+
+  private void startCamera() {
+    cameraHelper = new CameraXPreviewHelper();
+    cameraHelper.setOnCameraStartedListener(
+        surfaceTexture -> {
+          previewFrameTexture = surfaceTexture;
+          // Make the display view visible to start showing the preview. This triggers the
+          // SurfaceHolder.Callback added to (the holder of) previewDisplayView.
+          previewDisplayView.setVisibility(View.VISIBLE);
+        });
+    cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
+  }
+}

mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/res/layout/activity_main.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <FrameLayout
+        android:id="@+id/preview_display_layout"
+        android:layout_width="fill_parent"
+        android:layout_height="fill_parent"
+        android:layout_weight="1">
+        <TextView
+            android:id="@+id/no_camera_access_view"
+            android:layout_height="fill_parent"
+            android:layout_width="fill_parent"
+            android:gravity="center"
+            android:text="@string/no_camera_access" />
+    </FrameLayout>
+</androidx.constraintlayout.widget.ConstraintLayout>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/res/values/colors.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <color name="colorPrimary">#008577</color>
+    <color name="colorPrimaryDark">#00574B</color>
+    <color name="colorAccent">#D81B60</color>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/res/values/strings.xml

@@ -0,0 +1,4 @@
+<resources>
+    <string name="app_name" translatable="false">Template Matching CPU</string>
+    <string name="no_camera_access" translatable="false">Please grant camera permissions.</string>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/res/values/styles.xml

@@ -0,0 +1,11 @@
+<resources>
+
+    <!-- Base application theme. -->
+    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+        <!-- Customize your theme here. -->
+        <item name="colorPrimary">@color/colorPrimary</item>
+        <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
+        <item name="colorAccent">@color/colorAccent</item>
+    </style>
+
+</resources>

mediapipe/examples/desktop/autoflip/autoflip_messages.proto

@@ -150,4 +150,34 @@ message ConversionOptions {
   optional int32 target_height = 2;
 }
 
-// TODO: Move other autoflip messages into this area.
+// Self-contained message that provides all needed information to render
+// autoflip with an external renderer.  One of these messages is required for
+// each frame of the video.
+message ExternalRenderFrame {
+  // Rectangle using opencv standard.
+  message Rect {
+    optional float x = 1;
+    optional float y = 2;
+    optional float width = 3;
+    optional float height = 4;
+  }
+  // RGB color [0...255]
+  message Color {
+    optional int32 r = 1;
+    optional int32 g = 2;
+    optional int32 b = 3;
+  }
+  // Rect that must be cropped out of the input frame.  It is in the
+  // original dimensions of the input video.  The first step to render this
+  // frame is to crop this rect from the input frame.
+  optional Rect crop_from_location = 1;
+  // The placement location where the above rect is placed on the output frame.
+  // This will always have the same aspect ratio as the above rect but scaling
+  // may be required.
+  optional Rect render_to_location = 2;
+  // If render_to_location is smaller than the output dimensions of the frame,
+  // fill the rest of the frame with this color.
+  optional Color padding_color = 3;
+  // Timestamp in microseconds of this frame.
+  optional uint64 timestamp_us = 4;
+}

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc

@@ -44,11 +44,19 @@ constexpr char kInputExternalSettings[] = "EXTERNAL_SETTINGS";
 // TargetSizeType::MAXIMIZE_TARGET_DIMENSION
 constexpr char kAspectRatio[] = "EXTERNAL_ASPECT_RATIO";
 
+// Output the cropped frames, as well as visualization of crop regions and focus
+// points. Note that, KEY_FRAME_CROP_REGION_VIZ_FRAMES and
+// SALIENT_POINT_FRAME_VIZ_FRAMES can only be enabled when CROPPED_FRAMES is
+// enabled.
 constexpr char kOutputCroppedFrames[] = "CROPPED_FRAMES";
 constexpr char kOutputKeyFrameCropViz[] = "KEY_FRAME_CROP_REGION_VIZ_FRAMES";
 constexpr char kOutputFocusPointFrameViz[] = "SALIENT_POINT_FRAME_VIZ_FRAMES";
 constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
 
+// External rendering outputs
+constexpr char kExternalRenderingPerFrame[] = "EXTERNAL_RENDERING_PER_FRAME";
+constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
+
 ::mediapipe::Status SceneCroppingCalculator::GetContract(
     ::mediapipe::CalculatorContract* cc) {
   if (cc->InputSidePackets().HasTag(kInputExternalSettings)) {
@@ -67,16 +75,36 @@ constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
   }
   cc->Inputs().Tag(kInputShotBoundaries).Set<bool>();
 
-  cc->Outputs().Tag(kOutputCroppedFrames).Set<ImageFrame>();
+  if (cc->Outputs().HasTag(kOutputCroppedFrames)) {
+    cc->Outputs().Tag(kOutputCroppedFrames).Set<ImageFrame>();
+  }
   if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
+    RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames))
+        << "KEY_FRAME_CROP_REGION_VIZ_FRAMES can only be used when "
+           "CROPPED_FRAMES is specified.";
     cc->Outputs().Tag(kOutputKeyFrameCropViz).Set<ImageFrame>();
   }
   if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
+    RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames))
+        << "SALIENT_POINT_FRAME_VIZ_FRAMES can only be used when "
+           "CROPPED_FRAMES is specified.";
     cc->Outputs().Tag(kOutputFocusPointFrameViz).Set<ImageFrame>();
   }
   if (cc->Outputs().HasTag(kOutputSummary)) {
     cc->Outputs().Tag(kOutputSummary).Set<VideoCroppingSummary>();
   }
+  if (cc->Outputs().HasTag(kExternalRenderingPerFrame)) {
+    cc->Outputs().Tag(kExternalRenderingPerFrame).Set<ExternalRenderFrame>();
+  }
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    cc->Outputs()
+        .Tag(kExternalRenderingFullVid)
+        .Set<std::vector<ExternalRenderFrame>>();
+  }
+  RET_CHECK(cc->Outputs().HasTag(kExternalRenderingPerFrame) ||
+            cc->Outputs().HasTag(kExternalRenderingFullVid) ||
+            cc->Outputs().HasTag(kOutputCroppedFrames))
+      << "At leaset one output stream must be specified";
   return ::mediapipe::OkStatus();
 }
 
@@ -104,6 +132,11 @@ constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
   if (cc->Outputs().HasTag(kOutputSummary)) {
     summary_ = absl::make_unique<VideoCroppingSummary>();
   }
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    external_render_list_ =
+        absl::make_unique<std::vector<ExternalRenderFrame>>();
+  }
+  should_perform_frame_cropping_ = cc->Outputs().HasTag(kOutputCroppedFrames);
   return ::mediapipe::OkStatus();
 }
 
@@ -127,6 +160,28 @@ namespace {
   *aspect_ratio = width_ratio / height_ratio;
   return ::mediapipe::OkStatus();
 }
+void ConstructExternalRenderMessage(
+    const cv::Rect& crop_from_location, const cv::Rect& render_to_location,
+    const cv::Scalar& padding_color, const uint64 timestamp_us,
+    ExternalRenderFrame* external_render_message) {
+  auto crop_from_message =
+      external_render_message->mutable_crop_from_location();
+  crop_from_message->set_x(crop_from_location.x);
+  crop_from_message->set_y(crop_from_location.y);
+  crop_from_message->set_width(crop_from_location.width);
+  crop_from_message->set_height(crop_from_location.height);
+  auto render_to_message =
+      external_render_message->mutable_render_to_location();
+  render_to_message->set_x(render_to_location.x);
+  render_to_message->set_y(render_to_location.y);
+  render_to_message->set_width(render_to_location.width);
+  render_to_message->set_height(render_to_location.height);
+  auto padding_color_message = external_render_message->mutable_padding_color();
+  padding_color_message->set_r(padding_color[0]);
+  padding_color_message->set_g(padding_color[1]);
+  padding_color_message->set_b(padding_color[2]);
+  external_render_message->set_timestamp_us(timestamp_us);
+}
 }  // namespace
 
 ::mediapipe::Status SceneCroppingCalculator::Process(
@@ -230,8 +285,9 @@ namespace {
     is_end_of_scene = cc->Inputs().Tag(kInputShotBoundaries).Get<bool>();
   }
   const bool force_buffer_flush =
-      scene_frames_.size() >= options_.max_scene_size();
-  if (!scene_frames_.empty() && (is_end_of_scene || force_buffer_flush)) {
+      scene_frame_timestamps_.size() >= options_.max_scene_size();
+  if (!scene_frame_timestamps_.empty() &&
+      (is_end_of_scene || force_buffer_flush)) {
     MP_RETURN_IF_ERROR(ProcessScene(is_end_of_scene, cc));
   }
 
@@ -240,11 +296,14 @@ namespace {
     LOG_EVERY_N(ERROR, 10)
         << "------------------------ (Breathing) Time(s): "
         << cc->Inputs().Tag(kInputVideoFrames).Value().Timestamp().Seconds();
-    const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
-    const cv::Mat frame_mat = formats::MatView(&frame);
-    cv::Mat copy_mat;
-    frame_mat.copyTo(copy_mat);
-    scene_frames_.push_back(copy_mat);
+    // Only buffer frames if |should_perform_frame_cropping_| is true.
+    if (should_perform_frame_cropping_) {
+      const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
+      const cv::Mat frame_mat = formats::MatView(&frame);
+      cv::Mat copy_mat;
+      frame_mat.copyTo(copy_mat);
+      scene_frames_or_empty_.push_back(copy_mat);
+    }
     scene_frame_timestamps_.push_back(cc->InputTimestamp().Value());
     is_key_frames_.push_back(
         !cc->Inputs().Tag(kInputDetections).Value().IsEmpty());
@@ -274,7 +333,7 @@ namespace {
 
 ::mediapipe::Status SceneCroppingCalculator::Close(
     ::mediapipe::CalculatorContext* cc) {
-  if (!scene_frames_.empty()) {
+  if (!scene_frame_timestamps_.empty()) {
     MP_RETURN_IF_ERROR(ProcessScene(/* is_end_of_scene = */ true, cc));
   }
   if (cc->Outputs().HasTag(kOutputSummary)) {
@@ -282,16 +341,25 @@ namespace {
         .Tag(kOutputSummary)
         .Add(summary_.release(), Timestamp::PostStream());
   }
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    cc->Outputs()
+        .Tag(kExternalRenderingFullVid)
+        .Add(external_render_list_.release(), Timestamp::PostStream());
+  }
   return ::mediapipe::OkStatus();
 }
 
-::mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders() {
-  int top_border_size = 0, bottom_border_size = 0;
+// TODO: split this function into two, one for calculating the border
+// sizes, the other for the actual removal of borders from the frames.
+::mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders(
+    int* top_border_size, int* bottom_border_size) {
+  *top_border_size = 0;
+  *bottom_border_size = 0;
   MP_RETURN_IF_ERROR(ComputeSceneStaticBordersSize(
-      static_features_, &top_border_size, &bottom_border_size));
+      static_features_, top_border_size, bottom_border_size));
   const double scale = static_cast<double>(frame_height_) / key_frame_height_;
-  top_border_distance_ = std::round(scale * top_border_size);
-  const int bottom_border_distance = std::round(scale * bottom_border_size);
+  top_border_distance_ = std::round(scale * *top_border_size);
+  const int bottom_border_distance = std::round(scale * *bottom_border_size);
   effective_frame_height_ =
       frame_height_ - top_border_distance_ - bottom_border_distance;
 
@@ -301,10 +369,10 @@ namespace {
     // Remove borders from frames.
     cv::Rect roi(0, top_border_distance_, frame_width_,
                  effective_frame_height_);
-    for (int i = 0; i < scene_frames_.size(); ++i) {
+    for (int i = 0; i < scene_frames_or_empty_.size(); ++i) {
       cv::Mat tmp;
-      scene_frames_[i](roi).copyTo(tmp);
-      scene_frames_[i] = tmp;
+      scene_frames_or_empty_[i](roi).copyTo(tmp);
+      scene_frames_or_empty_[i] = tmp;
     }
     // Adjust detection bounding boxes.
     for (int i = 0; i < key_frame_infos_.size(); ++i) {
@@ -373,7 +441,9 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
   FilterKeyFrameInfo();
 
   // Removes any static borders.
-  MP_RETURN_IF_ERROR(RemoveStaticBorders());
+  int top_static_border_size, bottom_static_border_size;
+  MP_RETURN_IF_ERROR(
+      RemoveStaticBorders(&top_static_border_size, &bottom_static_border_size));
 
   // Decides if solid background color padding is possible and sets up color
   // interpolation functions in CIELAB. Uses linear interpolation by default.
@@ -409,21 +479,32 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
 
   // Crops scene frames.
   std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+
+  auto* cropped_frames_ptr =
+      should_perform_frame_cropping_ ? &cropped_frames : nullptr;
+
   MP_RETURN_IF_ERROR(scene_cropper_->CropFrames(
-      scene_summary, scene_frames_, focus_point_frames,
-      prior_focus_point_frames_, &cropped_frames));
+      scene_summary, scene_frame_timestamps_.size(), scene_frames_or_empty_,
+      focus_point_frames, prior_focus_point_frames_, top_static_border_size,
+      bottom_static_border_size, &crop_from_locations, cropped_frames_ptr));
 
   // Formats and outputs cropped frames.
   bool apply_padding = false;
   float vertical_fill_precent;
-  MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
-      cropped_frames, &apply_padding, &vertical_fill_precent, cc));
-
+  std::vector<cv::Rect> render_to_locations;
+  cv::Scalar padding_color;
+  if (should_perform_frame_cropping_) {
+    MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
+        cropped_frames, &render_to_locations, &apply_padding, &padding_color,
+        &vertical_fill_precent, cc));
+  }
   // Caches prior FocusPointFrames if this was not the end of a scene.
   prior_focus_point_frames_.clear();
   if (!is_end_of_scene) {
-    const int start = std::max(0, static_cast<int>(scene_frames_.size()) -
-                                      options_.prior_frame_buffer_size());
+    const int start =
+        std::max(0, static_cast<int>(scene_frame_timestamps_.size()) -
+                        options_.prior_frame_buffer_size());
     for (int i = start; i < num_key_frames; ++i) {
       prior_focus_point_frames_.push_back(focus_point_frames[i]);
     }
@@ -449,8 +530,31 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
     scene_summary->set_is_padded(apply_padding);
   }
 
+  if (cc->Outputs().HasTag(kExternalRenderingPerFrame)) {
+    for (int i = 0; i < scene_frame_timestamps_.size(); i++) {
+      auto external_render_message = absl::make_unique<ExternalRenderFrame>();
+      ConstructExternalRenderMessage(
+          crop_from_locations[i], render_to_locations[i], padding_color,
+          scene_frame_timestamps_[i], external_render_message.get());
+      cc->Outputs()
+          .Tag(kExternalRenderingPerFrame)
+          .Add(external_render_message.release(),
+               Timestamp(scene_frame_timestamps_[i]));
+    }
+  }
+
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    for (int i = 0; i < scene_frame_timestamps_.size(); i++) {
+      ExternalRenderFrame render_frame;
+      ConstructExternalRenderMessage(crop_from_locations[i],
+                                     render_to_locations[i], padding_color,
+                                     scene_frame_timestamps_[i], &render_frame);
+      external_render_list_->push_back(render_frame);
+    }
+  }
+
   key_frame_infos_.clear();
-  scene_frames_.clear();
+  scene_frames_or_empty_.clear();
   scene_frame_timestamps_.clear();
   is_key_frames_.clear();
   static_features_.clear();
@@ -459,8 +563,10 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
 }
 
 ::mediapipe::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
-    const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
-    float* vertical_fill_precent, CalculatorContext* cc) {
+    const std::vector<cv::Mat>& cropped_frames,
+    std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
+    cv::Scalar* padding_color, float* vertical_fill_precent,
+    CalculatorContext* cc) {
   RET_CHECK(apply_padding) << "Has padding boolean is null.";
   if (cropped_frames.empty()) {
     return ::mediapipe::OkStatus();
@@ -493,10 +599,22 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
             << " target height = " << target_height_;
   }
 
+  // Compute the "render to" location.  This is where the rect taken from the
+  // input video gets pasted on the output frame.  For use with external
+  // rendering solutions.
+  const int num_frames = cropped_frames.size();
+  for (int i = 0; i < num_frames; i++) {
+    if (*apply_padding) {
+      render_to_locations->push_back(padder_->ComputeOutputLocation());
+    } else {
+      render_to_locations->push_back(
+          cv::Rect(0, 0, target_width_, target_height_));
+    }
+  }
+
   // Resizes cropped frames, pads frames, and output frames.
   cv::Scalar* background_color = nullptr;
   cv::Scalar interpolated_color;
-  const int num_frames = cropped_frames.size();
   for (int i = 0; i < num_frames; ++i) {
     const int64 time_ms = scene_frame_timestamps_[i];
     const Timestamp timestamp(time_ms);
@@ -561,9 +679,9 @@ mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
   if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
     std::vector<std::unique_ptr<ImageFrame>> viz_frames;
     MP_RETURN_IF_ERROR(DrawDetectionsAndCropRegions(
-        scene_frames_, is_key_frames_, key_frame_infos_, key_frame_crop_results,
-        frame_format_, &viz_frames));
-    for (int i = 0; i < scene_frames_.size(); ++i) {
+        scene_frames_or_empty_, is_key_frames_, key_frame_infos_,
+        key_frame_crop_results, frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_or_empty_.size(); ++i) {
       cc->Outputs()
           .Tag(kOutputKeyFrameCropViz)
           .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
@@ -572,9 +690,10 @@ mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
   if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
     std::vector<std::unique_ptr<ImageFrame>> viz_frames;
     MP_RETURN_IF_ERROR(DrawFocusPointAndCropWindow(
-        scene_frames_, focus_point_frames, options_.viz_overlay_opacity(),
-        crop_window_width, crop_window_height, frame_format_, &viz_frames));
-    for (int i = 0; i < scene_frames_.size(); ++i) {
+        scene_frames_or_empty_, focus_point_frames,
+        options_.viz_overlay_opacity(), crop_window_width, crop_window_height,
+        frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_or_empty_.size(); ++i) {
       cc->Outputs()
           .Tag(kOutputFocusPointFrameViz)
           .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h

@@ -79,8 +79,10 @@ namespace autoflip {
 //     Indicators for shot boundaries (output of shot boundary detection).
 // - optional tag KEY_FRAMES (type ImageFrame):
 //     Key frames on which features are detected. This is only used to set the
-//     detection features frame size, and when it is omitted, the features frame
-//     size is assumed to be the original scene frame size.
+//     detection features frame size.  Alternatively, set
+//     video_feature_width/video_features_height within the options proto to
+//     define this value.  When neither is set, the features frame size is
+//     assumed to be the original scene frame size.
 //
 // Output streams:
 // - required tag CROPPED_FRAMES (type ImageFrame):
@@ -95,6 +97,12 @@ namespace autoflip {
 // - optional tag CROPPING_SUMMARY (type VideoCroppingSummary):
 //     Debug summary information for the video. Only generates one packet when
 //     calculator closes.
+// - optional tag EXTERNAL_RENDERING_PER_FRAME (type ExternalRenderFrame)
+//     Provides a per-frame message that can be used to render autoflip using an
+//     external renderer.
+// - optional tag EXTERNAL_RENDERING_FULL_VID (type Vector<ExternalRenderFrame>)
+//     Provides an end-stream message that can be used to render autoflip using
+//     an external renderer.
 //
 // Example config:
 // node {
@@ -134,8 +142,11 @@ class SceneCroppingCalculator : public CalculatorBase {
   ::mediapipe::Status Close(::mediapipe::CalculatorContext* cc) override;
 
  private:
-  // Removes any static borders from the scene frames before cropping.
-  ::mediapipe::Status RemoveStaticBorders();
+  // Removes any static borders from the scene frames before cropping. The
+  // arguments |top_border_size| and |bottom_border_size| report the size of the
+  // removed borders.
+  ::mediapipe::Status RemoveStaticBorders(int* top_border_size,
+                                          int* bottom_border_size);
 
   // Initializes a FrameCropRegionComputer given input and target frame sizes.
   ::mediapipe::Status InitializeFrameCropRegionComputer();
@@ -158,8 +169,10 @@ class SceneCroppingCalculator : public CalculatorBase {
   // solid background from static features if possible, otherwise uses blurred
   // background. Sets apply_padding to true if the scene is padded.
   ::mediapipe::Status FormatAndOutputCroppedFrames(
-      const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
-      float* vertical_fill_precent, CalculatorContext* cc);
+      const std::vector<cv::Mat>& cropped_frames,
+      std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
+      cv::Scalar* padding_color, float* vertical_fill_precent,
+      CalculatorContext* cc);
 
   // Draws and outputs visualization frames if those streams are present.
   ::mediapipe::Status OutputVizFrames(
@@ -193,7 +206,11 @@ class SceneCroppingCalculator : public CalculatorBase {
 
   // Buffered frames, timestamps, and indicators for key frames in the current
   // scene (size = number of input video frames).
-  std::vector<cv::Mat> scene_frames_;
+  // Note: scene_frames_or_empty_ may be empty if the actual cropping operation
+  // of frames is turned off, e.g. when |should_perform_frame_cropping_| is
+  // false, so rely on scene_frame_timestamps_.size() to query the number of
+  // accumulated timestamps rather than scene_frames_or_empty_.size().
+  std::vector<cv::Mat> scene_frames_or_empty_;
   std::vector<int64> scene_frame_timestamps_;
   std::vector<bool> is_key_frames_;
 
@@ -242,6 +259,17 @@ class SceneCroppingCalculator : public CalculatorBase {
 
   // Optional diagnostic summary output emitted in Close().
   std::unique_ptr<VideoCroppingSummary> summary_ = nullptr;
+
+  // Optional list of external rendering messages for each processed frame.
+  std::unique_ptr<std::vector<ExternalRenderFrame>> external_render_list_;
+
+  // Determines whether to perform real cropping on input frames. This flag is
+  // useful when the user only needs to compute cropping windows, in which case
+  // setting this flag to false can avoid buffering as well as cropping frames.
+  // This can significantly reduce memory usage and speed up processing. Some
+  // debugging visualization inevitably will be disabled because of this flag
+  // too.
+  bool should_perform_frame_cropping_ = false;
 };
 }  // namespace autoflip
 }  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc

@@ -68,6 +68,22 @@ constexpr char kNoKeyFrameConfig[] = R"(
     }
   })";
 
+constexpr char kDebugConfigNoCroppedFrame[] = R"(
+  calculator: "SceneCroppingCalculator"
+  input_stream: "VIDEO_FRAMES:camera_frames_org"
+  input_stream: "KEY_FRAMES:down_sampled_frames"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:border_features"
+  input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
+  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      target_width: $0
+      target_height: $1
+    }
+  })";
+
 constexpr char kDebugConfig[] = R"(
   calculator: "SceneCroppingCalculator"
   input_stream: "VIDEO_FRAMES:camera_frames_org"
@@ -79,6 +95,8 @@ constexpr char kDebugConfig[] = R"(
   output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
   output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
   output_stream: "CROPPING_SUMMARY:cropping_summaries"
+  output_stream: "EXTERNAL_RENDERING_PER_FRAME:external_rendering_per_frame"
+  output_stream: "EXTERNAL_RENDERING_FULL_VID:external_rendering_full_vid"
   options: {
     [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
       target_width: $0
@@ -257,6 +275,17 @@ TEST(SceneCroppingCalculatorTest, ChecksPriorFrameBufferSize) {
               HasSubstr("Prior frame buffer size is negative."));
 }
 
+TEST(SceneCroppingCalculatorTest, ChecksDebugConfigWithoutCroppedFrame) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kDebugConfigNoCroppedFrame, kTargetWidth, kTargetHeight,
+          kTargetSizeType, 0, kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  const auto status = runner->Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("can only be used when"));
+}
+
 // Checks that the calculator crops scene frames when there is no input key
 // frames stream.
 TEST(SceneCroppingCalculatorTest, HandlesNoKeyFrames) {
@@ -299,14 +328,34 @@ TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) {
   EXPECT_TRUE(outputs.HasTag("KEY_FRAME_CROP_REGION_VIZ_FRAMES"));
   EXPECT_TRUE(outputs.HasTag("SALIENT_POINT_FRAME_VIZ_FRAMES"));
   EXPECT_TRUE(outputs.HasTag("CROPPING_SUMMARY"));
+  EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_PER_FRAME"));
+  EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_FULL_VID"));
   const auto& crop_region_viz_frames_outputs =
       outputs.Tag("KEY_FRAME_CROP_REGION_VIZ_FRAMES").packets;
   const auto& salient_point_viz_frames_outputs =
       outputs.Tag("SALIENT_POINT_FRAME_VIZ_FRAMES").packets;
   const auto& summary_output = outputs.Tag("CROPPING_SUMMARY").packets;
+  const auto& ext_render_per_frame =
+      outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
+  const auto& ext_render_full_vid =
+      outputs.Tag("EXTERNAL_RENDERING_FULL_VID").packets;
   EXPECT_EQ(crop_region_viz_frames_outputs.size(), num_frames);
   EXPECT_EQ(salient_point_viz_frames_outputs.size(), num_frames);
   EXPECT_EQ(summary_output.size(), 1);
+  EXPECT_EQ(ext_render_per_frame.size(), num_frames);
+  EXPECT_EQ(ext_render_full_vid.size(), 1);
+  EXPECT_EQ(ext_render_per_frame[0].Get<ExternalRenderFrame>().timestamp_us(),
+            0);
+  EXPECT_EQ(ext_render_full_vid[0]
+                .Get<std::vector<ExternalRenderFrame>>()[0]
+                .timestamp_us(),
+            0);
+  EXPECT_EQ(ext_render_per_frame[1].Get<ExternalRenderFrame>().timestamp_us(),
+            20000);
+  EXPECT_EQ(ext_render_full_vid[0]
+                .Get<std::vector<ExternalRenderFrame>>()[1]
+                .timestamp_us(),
+            20000);
 
   for (int i = 0; i < num_frames; ++i) {
     const auto& crop_region_viz_frame =

mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.cc

@@ -173,5 +173,28 @@ PaddingEffectGenerator::PaddingEffectGenerator(const int input_width,
   return ::mediapipe::OkStatus();
 }
 
+cv::Rect PaddingEffectGenerator::ComputeOutputLocation() {
+  const int effective_input_width =
+      is_vertical_padding_ ? input_width_ : input_height_;
+  const int effective_input_height =
+      is_vertical_padding_ ? input_height_ : input_width_;
+  const int effective_output_width =
+      is_vertical_padding_ ? output_width_ : output_height_;
+  const int effective_output_height =
+      is_vertical_padding_ ? output_height_ : output_width_;
+
+  // Step 3 from "process" call above, compute foreground location.
+  const int foreground_height =
+      effective_input_height * effective_output_width / effective_input_width;
+  const int x = 0;
+  const int y = (effective_output_height - foreground_height) / 2;
+  const int width = effective_output_width;
+  const int height = foreground_height;
+
+  cv::Rect region_to_embed_foreground(x, y, width, height);
+
+  return region_to_embed_foreground;
+}
+
 }  // namespace autoflip
 }  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h

@@ -55,6 +55,10 @@ class PaddingEffectGenerator {
       ImageFrame* output_frame,
       const cv::Scalar* background_color_in_rgb = nullptr);
 
+  // Compute the "render location" on the output frame where the "crop from"
+  // location is to be placed.  For use with external rendering soutions.
+  cv::Rect ComputeOutputLocation();
+
  private:
   double target_aspect_ratio_;
   int input_width_ = -1;

mediapipe/examples/desktop/autoflip/quality/padding_effect_generator_test.cc

@@ -182,6 +182,16 @@ TEST(PaddingEffectGeneratorTest, ScaleToMultipleOfTwo) {
   EXPECT_EQ(result_frame.Width(), expect_width);
   EXPECT_EQ(result_frame.Height(), expect_height);
 }
+
+TEST(PaddingEffectGeneratorTest, ComputeOutputLocation) {
+  PaddingEffectGenerator generator(1920, 1080, 1.0);
+
+  auto result_rect = generator.ComputeOutputLocation();
+  EXPECT_EQ(result_rect.x, 0);
+  EXPECT_EQ(result_rect.y, 236);
+  EXPECT_EQ(result_rect.width, 1080);
+  EXPECT_EQ(result_rect.height, 607);
+}
 }  // namespace
 }  // namespace autoflip
 }  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_cropper.cc

@@ -25,14 +25,13 @@ namespace mediapipe {
 namespace autoflip {
 
 ::mediapipe::Status SceneCropper::CropFrames(
-    const SceneKeyFrameCropSummary& scene_summary,
-    const std::vector<cv::Mat>& scene_frames,
+    const SceneKeyFrameCropSummary& scene_summary, const int num_scene_frames,
+    const std::vector<cv::Mat>& scene_frames_or_empty,
     const std::vector<FocusPointFrame>& focus_point_frames,
     const std::vector<FocusPointFrame>& prior_focus_point_frames,
+    int top_static_border_size, int bottom_static_border_size,
+    std::vector<cv::Rect>* crop_from_location,
     std::vector<cv::Mat>* cropped_frames) const {
-  RET_CHECK_NE(cropped_frames, nullptr) << "Output cropped frames is null.";
-
-  const int num_scene_frames = scene_frames.size();
   RET_CHECK_GT(num_scene_frames, 0) << "No scene frames.";
   RET_CHECK_EQ(focus_point_frames.size(), num_scene_frames)
       << "Wrong size of FocusPointFrames.";
@@ -69,15 +68,36 @@ namespace autoflip {
     xform = affine_opencv;
   }
 
+  // If no cropped_frames is passed in, return directly.
+  if (!cropped_frames) {
+    return ::mediapipe::OkStatus();
+  }
+  RET_CHECK(!scene_frames_or_empty.empty())
+      << "If |cropped_frames| != nullptr, scene_frames_or_empty must not be "
+         "empty.";
   // Prepares cropped frames.
   cropped_frames->resize(num_scene_frames);
   for (int i = 0; i < num_scene_frames; ++i) {
-    (*cropped_frames)[i] =
-        cv::Mat::zeros(crop_height, crop_width, scene_frames[i].type());
+    (*cropped_frames)[i] = cv::Mat::zeros(crop_height, crop_width,
+                                          scene_frames_or_empty[i].type());
   }
 
-  return AffineRetarget(cv::Size(crop_width, crop_height), scene_frames,
-                        scene_frame_xforms, cropped_frames);
+  // Store the "crop from" location on the input frame for use with an external
+  // renderer.
+  for (int i = 0; i < num_scene_frames; i++) {
+    const int left = scene_frame_xforms[i].at<float>(0, 2);
+    const int right = left + crop_width;
+    const int top = top_static_border_size;
+    const int bottom =
+        top_static_border_size +
+        (crop_height - top_static_border_size - bottom_static_border_size);
+    crop_from_location->push_back(
+        cv::Rect(left, top, right - left, bottom - top));
+  }
+
+  return AffineRetarget(cv::Size(crop_width, crop_height),
+                        scene_frames_or_empty, scene_frame_xforms,
+                        cropped_frames);
 }
 
 }  // namespace autoflip

mediapipe/examples/desktop/autoflip/quality/scene_cropper.h

@@ -48,14 +48,19 @@ class SceneCropper {
   SceneCropper() {}
   ~SceneCropper() {}
 
-  // Crops scene frames given SceneKeyFrameCropSummary, FocusPointFrames, and
-  // any prior FocusPointFrames (to ensure smoothness when there was no actual
-  // scene change).
+  // Computes transformation matrix given SceneKeyFrameCropSummary,
+  // FocusPointFrames, and any prior FocusPointFrames (to ensure smoothness when
+  // there was no actual scene change). Optionally crops the input frames based
+  // on the transform matrix if |cropped_frames| is not nullptr and
+  // |scene_frames_or_empty| isn't empty.
+  // TODO: split this function into two separate functions.
   ::mediapipe::Status CropFrames(
-      const SceneKeyFrameCropSummary& scene_summary,
-      const std::vector<cv::Mat>& scene_frames,
+      const SceneKeyFrameCropSummary& scene_summary, const int num_scene_frames,
+      const std::vector<cv::Mat>& scene_frames_or_empty,
       const std::vector<FocusPointFrame>& focus_point_frames,
       const std::vector<FocusPointFrame>& prior_focus_point_frames,
+      int top_static_border_size, int bottom_static_border_size,
+      std::vector<cv::Rect>* all_scene_frame_xforms,
       std::vector<cv::Mat>* cropped_frames) const;
 };
 

mediapipe/examples/desktop/autoflip/quality/scene_cropper_test.cc

@@ -71,24 +71,16 @@ std::vector<FocusPointFrame> GetDefaultFocusPointFrames() {
   return GetFocusPointFrames(kNumSceneFrames);
 }
 
-// Checks that CropFrames checks output pointer is not null.
-TEST(SceneCropperTest, CropFramesChecksOutputNotNull) {
-  SceneCropper scene_cropper;
-  const auto status = scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), nullptr);
-  EXPECT_FALSE(status.ok());
-  EXPECT_THAT(status.ToString(), HasSubstr("Output cropped frames is null."));
-}
-
 // Checks that CropFrames checks that scene frames size is positive.
 TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) {
   SceneCropper scene_cropper;
   std::vector<cv::Mat> scene_frames(0);
   std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
   const auto status = scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), scene_frames,
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames);
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
   EXPECT_FALSE(status.ok());
   EXPECT_THAT(status.ToString(), HasSubstr("No scene frames."));
 }
@@ -97,10 +89,12 @@ TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) {
 TEST(SceneCropperTest, CropFramesChecksFocusPointFramesSize) {
   SceneCropper scene_cropper;
   std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
   const auto status = scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0),
-      &cropped_frames);
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
   EXPECT_FALSE(status.ok());
   EXPECT_THAT(status.ToString(), HasSubstr("Wrong size of FocusPointFrames"));
 }
@@ -111,9 +105,12 @@ TEST(SceneCropperTest, CropFramesChecksCropSizePositive) {
   scene_summary.set_crop_window_width(-1);
   SceneCropper scene_cropper;
   std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
   const auto status = scene_cropper.CropFrames(
-      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
-      GetFocusPointFrames(0), &cropped_frames);
+      scene_summary, scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
   EXPECT_FALSE(status.ok());
   EXPECT_THAT(status.ToString(), HasSubstr("Crop width is non-positive."));
 }
@@ -124,9 +121,12 @@ TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
   scene_summary.set_crop_window_height(kSceneHeight + 1);
   SceneCropper scene_cropper;
   std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
   const auto status = scene_cropper.CropFrames(
-      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
-      GetFocusPointFrames(0), &cropped_frames);
+      scene_summary, scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
   EXPECT_FALSE(status.ok());
   EXPECT_THAT(status.ToString(),
               HasSubstr("Crop height exceeds frame height."));
@@ -136,9 +136,12 @@ TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
 TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
   SceneCropper scene_cropper;
   std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
   MP_ASSERT_OK(scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames));
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames));
   ASSERT_EQ(cropped_frames.size(), kNumSceneFrames);
   for (int i = 0; i < kNumSceneFrames; ++i) {
     EXPECT_EQ(cropped_frames[i].rows, kCropHeight);
@@ -150,9 +153,12 @@ TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
 TEST(SceneCropperTest, CropFramesWorksWithPriorFocusPointFrames) {
   SceneCropper scene_cropper;
   std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
   MP_EXPECT_OK(scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(3), &cropped_frames));
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(3), 0, 0,
+      &crop_from_locations, &cropped_frames));
   EXPECT_EQ(cropped_frames.size(), kNumSceneFrames);
   for (int i = 0; i < kNumSceneFrames; ++i) {
     EXPECT_EQ(cropped_frames[i].rows, kCropHeight);

mediapipe/examples/desktop/face_mesh/BUILD

@@ -0,0 +1,42 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "face_mesh_tflite",
+    deps = [
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/graphs/face_mesh:desktop_calculators",
+    ],
+)
+
+cc_binary(
+    name = "face_mesh_cpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/face_mesh:desktop_live_calculators",
+    ],
+)
+
+# Linux only
+cc_binary(
+    name = "face_mesh_gpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
+        "//mediapipe/graphs/face_mesh:desktop_live_gpu_calculators",
+    ],
+)

mediapipe/examples/desktop/template_matching/BUILD

@@ -0,0 +1,25 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "template_matching_tflite",
+    deps = [
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/graphs/template_matching:desktop_calculators",
+    ],
+)

mediapipe/examples/ios/facemeshgpu/AppDelegate.h

@@ -0,0 +1,21 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow *window;
+
+@end

mediapipe/examples/ios/facemeshgpu/AppDelegate.m

@@ -0,0 +1,59 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+
+- (BOOL)application:(UIApplication *)application
+    didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+  // Override point for customization after application launch.
+  return YES;
+}
+
+- (void)applicationWillResignActive:(UIApplication *)application {
+  // Sent when the application is about to move from active to inactive state. This can occur for
+  // certain types of temporary interruptions (such as an incoming phone call or SMS message) or
+  // when the user quits the application and it begins the transition to the background state. Use
+  // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering
+  // callbacks. Games should use this method to pause the game.
+}
+
+- (void)applicationDidEnterBackground:(UIApplication *)application {
+  // Use this method to release shared resources, save user data, invalidate timers, and store
+  // enough application state information to restore your application to its current state in case
+  // it is terminated later. If your application supports background execution, this method is
+  // called instead of applicationWillTerminate: when the user quits.
+}
+
+- (void)applicationWillEnterForeground:(UIApplication *)application {
+  // Called as part of the transition from the background to the active state; here you can undo
+  // many of the changes made on entering the background.
+}
+
+- (void)applicationDidBecomeActive:(UIApplication *)application {
+  // Restart any tasks that were paused (or not yet started) while the application was inactive. If
+  // the application was previously in the background, optionally refresh the user interface.
+}
+
+- (void)applicationWillTerminate:(UIApplication *)application {
+  // Called when the application is about to terminate. Save data if appropriate. See also
+  // applicationDidEnterBackground:.
+}
+
+@end

mediapipe/examples/ios/facemeshgpu/Assets.xcassets/AppIcon.appiconset/Contents.json

@@ -0,0 +1,99 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "83.5x83.5",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "size" : "1024x1024",
+      "scale" : "1x"
+    }
+  ],
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
+

mediapipe/examples/ios/facemeshgpu/Assets.xcassets/Contents.json

@@ -0,0 +1,7 @@
+{
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
+

mediapipe/examples/ios/facemeshgpu/BUILD

@@ -0,0 +1,76 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "@build_bazel_rules_apple//apple:ios.bzl",
+    "ios_application",
+)
+
+licenses(["notice"])  # Apache 2.0
+
+MIN_IOS_VERSION = "10.0"
+
+ios_application(
+    name = "FaceMeshGpuApp",
+    bundle_id = "com.google.mediapipe.FaceMeshGpu",
+    families = [
+        "iphone",
+        "ipad",
+    ],
+    infoplists = ["Info.plist"],
+    minimum_os_version = MIN_IOS_VERSION,
+    provisioning_profile = "//mediapipe/examples/ios:provisioning_profile",
+    deps = [
+        ":FaceMeshGpuAppLibrary",
+        "@ios_opencv//:OpencvFramework",
+    ],
+)
+
+objc_library(
+    name = "FaceMeshGpuAppLibrary",
+    srcs = [
+        "AppDelegate.m",
+        "ViewController.mm",
+        "main.m",
+    ],
+    hdrs = [
+        "AppDelegate.h",
+        "ViewController.h",
+    ],
+    data = [
+        "Base.lproj/LaunchScreen.storyboard",
+        "Base.lproj/Main.storyboard",
+        "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu_binary_graph",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+    ],
+    sdk_frameworks = [
+        "AVFoundation",
+        "CoreGraphics",
+        "CoreMedia",
+        "UIKit",
+    ],
+    deps = [
+        "//mediapipe/objc:mediapipe_framework_ios",
+        "//mediapipe/objc:mediapipe_input_sources_ios",
+        "//mediapipe/objc:mediapipe_layer_renderer",
+    ] + select({
+        "//mediapipe:ios_i386": [],
+        "//mediapipe:ios_x86_64": [],
+        "//conditions:default": [
+            "//mediapipe/graphs/face_mesh:mobile_calculators",
+            "//mediapipe/framework/formats:landmark_cc_proto",
+        ],
+    }),
+)

mediapipe/examples/ios/facemeshgpu/Base.lproj/LaunchScreen.storyboard

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>

mediapipe/examples/ios/facemeshgpu/Base.lproj/Main.storyboard

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14490.70" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14490.49"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="ViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
+                                <rect key="frame" x="0.0" y="20" width="375" height="647"/>
+                                <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                                <subviews>
+                                    <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
+                                        <rect key="frame" x="57" y="248" width="260" height="151"/>
+                                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
+                                        <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                        <color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <nil key="highlightedColor"/>
+                                    </label>
+                                </subviews>
+                                <color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                <accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
+                                    <bool key="isElement" value="YES"/>
+                                </accessibility>
+                            </view>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                    <connections>
+                        <outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
+                        <outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="48.799999999999997" y="20.239880059970016"/>
+        </scene>
+    </scenes>
+</document>

mediapipe/examples/ios/facemeshgpu/Info.plist

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>NSCameraUsageDescription</key>
+  <string>This app uses the camera to demonstrate live video processing.</string>
+  <key>CFBundleDevelopmentRegion</key>
+  <string>en</string>
+  <key>CFBundleExecutable</key>
+  <string>$(EXECUTABLE_NAME)</string>
+  <key>CFBundleIdentifier</key>
+  <string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+  <key>CFBundleInfoDictionaryVersion</key>
+  <string>6.0</string>
+  <key>CFBundleName</key>
+  <string>$(PRODUCT_NAME)</string>
+  <key>CFBundlePackageType</key>
+  <string>APPL</string>
+  <key>CFBundleShortVersionString</key>
+  <string>1.0</string>
+  <key>CFBundleVersion</key>
+  <string>1</string>
+  <key>LSRequiresIPhoneOS</key>
+  <true/>
+  <key>UILaunchStoryboardName</key>
+  <string>LaunchScreen</string>
+  <key>UIMainStoryboardFile</key>
+  <string>Main</string>
+  <key>UIRequiredDeviceCapabilities</key>
+  <array>
+    <string>armv7</string>
+  </array>
+  <key>UISupportedInterfaceOrientations</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+  <key>UISupportedInterfaceOrientations~ipad</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+</dict>
+</plist>

mediapipe/examples/ios/facemeshgpu/ViewController.h

@@ -0,0 +1,19 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface ViewController : UIViewController
+
+@end

mediapipe/examples/ios/facemeshgpu/ViewController.mm

@@ -0,0 +1,210 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "ViewController.h"
+
+#import "mediapipe/objc/MPPCameraInputSource.h"
+#import "mediapipe/objc/MPPGraph.h"
+#import "mediapipe/objc/MPPLayerRenderer.h"
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+static NSString* const kGraphName = @"face_mesh_mobile_gpu";
+
+static const char* kInputStream = "input_video";
+static const char* kNumFacesInputSidePacket = "num_faces";
+static const char* kOutputStream = "output_video";
+static const char* kLandmarksOutputStream = "multi_face_landmarks";
+static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
+
+// Max number of faces to detect/process.
+static const int kNumFaces = 1;
+
+@interface ViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
+
+// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and
+// sent video frames on _videoQueue.
+@property(nonatomic) MPPGraph* mediapipeGraph;
+
+@end
+
+@implementation ViewController {
+  /// Handles camera access via AVCaptureSession library.
+  MPPCameraInputSource* _cameraSource;
+
+  /// Inform the user when camera is unavailable.
+  IBOutlet UILabel* _noCameraLabel;
+  /// Display the camera preview frames.
+  IBOutlet UIView* _liveView;
+  /// Render frames in a layer.
+  MPPLayerRenderer* _renderer;
+
+  /// Process camera frames on this queue.
+  dispatch_queue_t _videoQueue;
+}
+
+#pragma mark - Cleanup methods
+
+- (void)dealloc {
+  self.mediapipeGraph.delegate = nil;
+  [self.mediapipeGraph cancel];
+  // Ignore errors since we're cleaning up.
+  [self.mediapipeGraph closeAllInputStreamsWithError:nil];
+  [self.mediapipeGraph waitUntilDoneWithError:nil];
+}
+
+#pragma mark - MediaPipe graph methods
+
++ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
+  // Load the graph config resource.
+  NSError* configLoadError = nil;
+  NSBundle* bundle = [NSBundle bundleForClass:[self class]];
+  if (!resource || resource.length == 0) {
+    return nil;
+  }
+  NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
+  NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
+  if (!data) {
+    NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError);
+    return nil;
+  }
+
+  // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object.
+  mediapipe::CalculatorGraphConfig config;
+  config.ParseFromArray(data.bytes, data.length);
+
+  // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
+  MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
+  [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
+  [newGraph addFrameOutputStream:kLandmarksOutputStream outputPacketType:MPPPacketTypeRaw];
+  [newGraph setSidePacket:(mediapipe::MakePacket<int>(kNumFaces)) named:kNumFacesInputSidePacket];
+  return newGraph;
+}
+
+#pragma mark - UIViewController methods
+
+- (void)viewDidLoad {
+  [super viewDidLoad];
+
+  _renderer = [[MPPLayerRenderer alloc] init];
+  _renderer.layer.frame = _liveView.layer.bounds;
+  [_liveView.layer addSublayer:_renderer.layer];
+  _renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop;
+  // When using the front camera, mirror the input for a more natural look.
+  _renderer.mirrored = YES;
+
+  dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class(
+      DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0);
+  _videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute);
+
+  _cameraSource = [[MPPCameraInputSource alloc] init];
+  [_cameraSource setDelegate:self queue:_videoQueue];
+  _cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
+  _cameraSource.cameraPosition = AVCaptureDevicePositionFront;
+  // The frame's native format is rotated with respect to the portrait orientation.
+  _cameraSource.orientation = AVCaptureVideoOrientationPortrait;
+
+  self.mediapipeGraph = [[self class] loadGraphFromResource:kGraphName];
+  self.mediapipeGraph.delegate = self;
+  // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
+  self.mediapipeGraph.maxFramesInFlight = 2;
+}
+
+// In this application, there is only one ViewController which has no navigation to other view
+// controllers, and there is only one View with live display showing the result of running the
+// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph
+// setup/teardown and camera start/stop logic should be updated appropriately in response to the
+// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times
+// depending on the application navigation flow in that case.
+- (void)viewWillAppear:(BOOL)animated {
+  [super viewWillAppear:animated];
+
+  [_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
+    if (granted) {
+      [self startGraphAndCamera];
+      dispatch_async(dispatch_get_main_queue(), ^{
+        _noCameraLabel.hidden = YES;
+      });
+    }
+  }];
+}
+
+- (void)startGraphAndCamera {
+  // Start running self.mediapipeGraph.
+  NSError* error;
+  if (![self.mediapipeGraph startWithError:&error]) {
+    NSLog(@"Failed to start graph: %@", error);
+  }
+
+  // Start fetching frames from the camera.
+  dispatch_async(_videoQueue, ^{
+    [_cameraSource start];
+  });
+}
+
+#pragma mark - MPPGraphDelegate methods
+
+// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+    didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
+              fromStream:(const std::string&)streamName {
+  if (streamName == kOutputStream) {
+    // Display the captured image on the screen.
+    CVPixelBufferRetain(pixelBuffer);
+    dispatch_async(dispatch_get_main_queue(), ^{
+      [_renderer renderPixelBuffer:pixelBuffer];
+      CVPixelBufferRelease(pixelBuffer);
+    });
+  }
+}
+
+// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+     didOutputPacket:(const ::mediapipe::Packet&)packet
+          fromStream:(const std::string&)streamName {
+  if (streamName == kLandmarksOutputStream) {
+    if (packet.IsEmpty()) {
+      NSLog(@"[TS:%lld] No face landmarks", packet.Timestamp().Value());
+      return;
+    }
+    const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
+    NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(),
+          multi_face_landmarks.size());
+    for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) {
+      const auto& landmarks = multi_face_landmarks[face_index];
+      NSLog(@"\tNumber of landmarks for face[%d]: %d", face_index, landmarks.landmark_size());
+      for (int i = 0; i < landmarks.landmark_size(); ++i) {
+        NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(),
+              landmarks.landmark(i).y(), landmarks.landmark(i).z());
+      }
+    }
+  }
+}
+
+#pragma mark - MPPInputSourceDelegate methods
+
+// Must be invoked on _videoQueue.
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
+                timestamp:(CMTime)timestamp
+               fromSource:(MPPInputSource*)source {
+  if (source != _cameraSource) {
+    NSLog(@"Unknown source: %@", source);
+    return;
+  }
+  [self.mediapipeGraph sendPixelBuffer:imageBuffer
+                            intoStream:kInputStream
+                            packetType:MPPPacketTypePixelBuffer];
+}
+
+@end

mediapipe/examples/ios/facemeshgpu/main.m

@@ -0,0 +1,22 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char * argv[]) {
+  @autoreleasepool {
+      return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
+  }
+}