Project import generated by Copybara.

GitOrigin-RevId: 4cee4a2c2317fb190680c17e31ebbb03bb73b71c

mediapipe/modules/face_geometry/BUILD

@@ -0,0 +1,116 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
+load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph")
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+mediapipe_simple_subgraph(
+    name = "face_geometry",
+    graph = "face_geometry.pbtxt",
+    register_as = "FaceGeometry",
+    deps = [
+        ":geometry_pipeline_calculator",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "effect_renderer_calculator_proto",
+    srcs = ["effect_renderer_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+    ],
+)
+
+cc_library(
+    name = "effect_renderer_calculator",
+    srcs = ["effect_renderer_calculator.cc"],
+    deps = [
+        ":effect_renderer_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgcodecs",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "//mediapipe/gpu:gl_calculator_helper",
+        "//mediapipe/gpu:gpu_buffer",
+        "//mediapipe/modules/face_geometry/libs:effect_renderer",
+        "//mediapipe/modules/face_geometry/libs:validation_utils",
+        "//mediapipe/modules/face_geometry/protos:environment_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
+        "//mediapipe/util:resource_util",
+        "@com_google_absl//absl/types:optional",
+    ],
+    alwayslink = 1,
+)
+
+mediapipe_proto_library(
+    name = "env_generator_calculator_proto",
+    srcs = ["env_generator_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/modules/face_geometry/protos:environment_proto",
+    ],
+)
+
+cc_library(
+    name = "env_generator_calculator",
+    srcs = ["env_generator_calculator.cc"],
+    deps = [
+        ":env_generator_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/modules/face_geometry/libs:validation_utils",
+        "//mediapipe/modules/face_geometry/protos:environment_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
+mediapipe_proto_library(
+    name = "geometry_pipeline_calculator_proto",
+    srcs = ["geometry_pipeline_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+    ],
+)
+
+cc_library(
+    name = "geometry_pipeline_calculator",
+    srcs = ["geometry_pipeline_calculator.cc"],
+    deps = [
+        ":geometry_pipeline_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "//mediapipe/modules/face_geometry/libs:geometry_pipeline",
+        "//mediapipe/modules/face_geometry/libs:validation_utils",
+        "//mediapipe/modules/face_geometry/protos:environment_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
+        "//mediapipe/util:resource_util",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)

mediapipe/modules/face_geometry/README.md

@@ -0,0 +1,18 @@
+# face_geometry
+
+Protos|Details
+:--- | :---
+[`face_geometry.Environment`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/environment.proto)| Describes an environment; includes the camera frame origin point location as well as virtual camera parameters.
+[`face_geometry.GeometryPipelineMetadata`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.proto)| Describes metadata needed to estimate face geometry based on the face landmark module result.
+[`face_geometry.FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto)| Describes geometry data for a single face; includes a face mesh surface and a face pose in a given environment.
+[`face_geometry.Mesh3d`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/mesh_3d.proto)| Describes a 3D mesh surface.
+
+Calculators|Details
+:--- | :---
+[`FaceGeometryEnvGeneratorCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/env_generator_calculator.cc)| Generates an environment that describes a virtual scene.
+[`FaceGeometryPipelineCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc)| Extracts face geometry for multiple faces from a vector of landmark lists.
+[`FaceGeometryEffectRendererCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/effect_renderer_calculator.cc)| Renders a face effect.
+
+Subgraphs|Details
+:--- | :---
+[`FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry.pbtxt)| Extracts face geometry from landmarks for multiple faces.

mediapipe/modules/face_geometry/data/BUILD

@@ -0,0 +1,37 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+encode_binary_proto(
+    name = "geometry_pipeline_metadata",
+    input = "geometry_pipeline_metadata.pbtxt",
+    message_type = "mediapipe.face_geometry.GeometryPipelineMetadata",
+    output = "geometry_pipeline_metadata.binarypb",
+    deps = [
+        "//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_proto",
+    ],
+)
+
+# These canonical face model files are not meant to be used in runtime, but rather for asset
+# creation and/or reference.
+exports_files([
+    "canonical_face_model.fbx",
+    "canonical_face_model.obj",
+    "canonical_face_model_uv_visualization.png",
+])

mediapipe/modules/face_geometry/effect_renderer_calculator.cc

@@ -0,0 +1,284 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"       // NOTYPO
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"  // NOTYPO
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"    // NOTYPO
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#include "mediapipe/gpu/gpu_buffer.h"
+#include "mediapipe/modules/face_geometry/effect_renderer_calculator.pb.h"
+#include "mediapipe/modules/face_geometry/libs/effect_renderer.h"
+#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
+#include "mediapipe/util/resource_util.h"
+
+namespace mediapipe {
+namespace {
+
+static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
+static constexpr char kImageGpuTag[] = "IMAGE_GPU";
+static constexpr char kMultiFaceGeometryTag[] = "MULTI_FACE_GEOMETRY";
+
+// A calculator that renders a visual effect for multiple faces.
+//
+// Inputs:
+//   IMAGE_GPU (`GpuBuffer`, required):
+//     A buffer containing input image.
+//
+//   MULTI_FACE_GEOMETRY (`std::vector<face_geometry::FaceGeometry>`, optional):
+//     A vector of face geometry data.
+//
+//     If absent, the input GPU buffer is copied over into the output GPU buffer
+//     without any effect being rendered.
+//
+// Input side packets:
+//   ENVIRONMENT (`face_geometry::Environment`, required)
+//     Describes an environment; includes the camera frame origin point location
+//     as well as virtual camera parameters.
+//
+// Output:
+//   IMAGE_GPU (`GpuBuffer`, required):
+//     A buffer with a visual effect being rendered for multiple faces.
+//
+// Options:
+//   effect_texture_path (`string`, required):
+//     Defines a path for the visual effect texture file. The effect texture is
+//     later rendered on top of the effect mesh.
+//
+//     The texture file format must be supported by the OpenCV image decoder. It
+//     must also define either an RGB or an RGBA texture.
+//
+//   effect_mesh_3d_path (`string`, optional):
+//     Defines a path for the visual effect mesh 3D file. The effect mesh is
+//     later "attached" to the face and is driven by the face pose
+//     transformation matrix.
+//
+//     The mesh 3D file format must be the binary `face_geometry.Mesh3d` proto.
+//
+//     If is not present, the runtime face mesh will be used as the effect mesh
+//     - this mode is handy for facepaint effects.
+//
+class EffectRendererCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc))
+        << "Failed to update contract for the GPU helper!";
+
+    cc->InputSidePackets()
+        .Tag(kEnvironmentTag)
+        .Set<face_geometry::Environment>();
+    cc->Inputs().Tag(kImageGpuTag).Set<GpuBuffer>();
+    cc->Inputs()
+        .Tag(kMultiFaceGeometryTag)
+        .Set<std::vector<face_geometry::FaceGeometry>>();
+    cc->Outputs().Tag(kImageGpuTag).Set<GpuBuffer>();
+
+    return mediapipe::GlCalculatorHelper::UpdateContract(cc);
+  }
+
+  mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(mediapipe::TimestampDiff(0));
+
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc))
+        << "Failed to open the GPU helper!";
+    return gpu_helper_.RunInGlContext([&]() -> mediapipe::Status {
+      const auto& options =
+          cc->Options<FaceGeometryEffectRendererCalculatorOptions>();
+
+      const auto& environment = cc->InputSidePackets()
+                                    .Tag(kEnvironmentTag)
+                                    .Get<face_geometry::Environment>();
+
+      MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
+          << "Invalid environment!";
+
+      absl::optional<face_geometry::Mesh3d> effect_mesh_3d;
+      if (options.has_effect_mesh_3d_path()) {
+        ASSIGN_OR_RETURN(effect_mesh_3d,
+                         ReadMesh3dFromFile(options.effect_mesh_3d_path()),
+                         _ << "Failed to read the effect 3D mesh from file!");
+
+        MP_RETURN_IF_ERROR(face_geometry::ValidateMesh3d(*effect_mesh_3d))
+            << "Invalid effect 3D mesh!";
+      }
+
+      ASSIGN_OR_RETURN(ImageFrame effect_texture,
+                       ReadTextureFromFile(options.effect_texture_path()),
+                       _ << "Failed to read the effect texture from file!");
+
+      ASSIGN_OR_RETURN(effect_renderer_,
+                       CreateEffectRenderer(environment, effect_mesh_3d,
+                                            std::move(effect_texture)),
+                       _ << "Failed to create the effect renderer!");
+
+      return mediapipe::OkStatus();
+    });
+  }
+
+  mediapipe::Status Process(CalculatorContext* cc) override {
+    // The `IMAGE_GPU` stream is required to have a non-empty packet. In case
+    // this requirement is not met, there's nothing to be processed at the
+    // current timestamp.
+    if (cc->Inputs().Tag(kImageGpuTag).IsEmpty()) {
+      return mediapipe::OkStatus();
+    }
+
+    return gpu_helper_.RunInGlContext([this, cc]() -> mediapipe::Status {
+      const auto& input_gpu_buffer =
+          cc->Inputs().Tag(kImageGpuTag).Get<GpuBuffer>();
+
+      GlTexture input_gl_texture =
+          gpu_helper_.CreateSourceTexture(input_gpu_buffer);
+
+      GlTexture output_gl_texture = gpu_helper_.CreateDestinationTexture(
+          input_gl_texture.width(), input_gl_texture.height());
+
+      std::vector<face_geometry::FaceGeometry> empty_multi_face_geometry;
+      const auto& multi_face_geometry =
+          cc->Inputs().Tag(kMultiFaceGeometryTag).IsEmpty()
+              ? empty_multi_face_geometry
+              : cc->Inputs()
+                    .Tag(kMultiFaceGeometryTag)
+                    .Get<std::vector<face_geometry::FaceGeometry>>();
+
+      // Validate input multi face geometry data.
+      for (const face_geometry::FaceGeometry& face_geometry :
+           multi_face_geometry) {
+        MP_RETURN_IF_ERROR(face_geometry::ValidateFaceGeometry(face_geometry))
+            << "Invalid face geometry!";
+      }
+
+      MP_RETURN_IF_ERROR(effect_renderer_->RenderEffect(
+          multi_face_geometry, input_gl_texture.width(),
+          input_gl_texture.height(), input_gl_texture.target(),
+          input_gl_texture.name(), output_gl_texture.target(),
+          output_gl_texture.name()))
+          << "Failed to render the effect!";
+
+      std::unique_ptr<GpuBuffer> output_gpu_buffer =
+          output_gl_texture.GetFrame<GpuBuffer>();
+
+      cc->Outputs()
+          .Tag(kImageGpuTag)
+          .AddPacket(mediapipe::Adopt<GpuBuffer>(output_gpu_buffer.release())
+                         .At(cc->InputTimestamp()));
+
+      output_gl_texture.Release();
+      input_gl_texture.Release();
+
+      return mediapipe::OkStatus();
+    });
+  }
+
+  ~EffectRendererCalculator() {
+    gpu_helper_.RunInGlContext([this]() { effect_renderer_.reset(); });
+  }
+
+ private:
+  static mediapipe::StatusOr<ImageFrame> ReadTextureFromFile(
+      const std::string& texture_path) {
+    ASSIGN_OR_RETURN(std::string texture_blob,
+                     ReadContentBlobFromFile(texture_path),
+                     _ << "Failed to read texture blob from file!");
+
+    // Use OpenCV image decoding functionality to finish reading the texture.
+    std::vector<char> texture_blob_vector(texture_blob.begin(),
+                                          texture_blob.end());
+    cv::Mat decoded_mat =
+        cv::imdecode(texture_blob_vector, cv::IMREAD_UNCHANGED);
+
+    RET_CHECK(decoded_mat.type() == CV_8UC3 || decoded_mat.type() == CV_8UC4)
+        << "Texture must have `char` as the underlying type and "
+           "must have either 3 or 4 channels!";
+
+    ImageFormat::Format image_format = ImageFormat::UNKNOWN;
+    cv::Mat output_mat;
+    switch (decoded_mat.channels()) {
+      case 3:
+        image_format = ImageFormat::SRGB;
+        cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGB);
+        break;
+
+      case 4:
+        image_format = ImageFormat::SRGBA;
+        cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGRA2RGBA);
+        break;
+
+      default:
+        RET_CHECK_FAIL()
+            << "Unexpected number of channels; expected 3 or 4, got "
+            << decoded_mat.channels() << "!";
+    }
+
+    ImageFrame output_image_frame(image_format, output_mat.size().width,
+                                  output_mat.size().height,
+                                  ImageFrame::kGlDefaultAlignmentBoundary);
+
+    output_mat.copyTo(formats::MatView(&output_image_frame));
+
+    return output_image_frame;
+  }
+
+  static mediapipe::StatusOr<face_geometry::Mesh3d> ReadMesh3dFromFile(
+      const std::string& mesh_3d_path) {
+    ASSIGN_OR_RETURN(std::string mesh_3d_blob,
+                     ReadContentBlobFromFile(mesh_3d_path),
+                     _ << "Failed to read mesh 3D blob from file!");
+
+    face_geometry::Mesh3d mesh_3d;
+    RET_CHECK(mesh_3d.ParseFromString(mesh_3d_blob))
+        << "Failed to parse a mesh 3D proto from a binary blob!";
+
+    return mesh_3d;
+  }
+
+  static mediapipe::StatusOr<std::string> ReadContentBlobFromFile(
+      const std::string& unresolved_path) {
+    ASSIGN_OR_RETURN(std::string resolved_path,
+                     mediapipe::PathToResourceAsFile(unresolved_path),
+                     _ << "Failed to resolve path! Path = " << unresolved_path);
+
+    std::string content_blob;
+    MP_RETURN_IF_ERROR(
+        mediapipe::GetResourceContents(resolved_path, &content_blob))
+        << "Failed to read content blob! Resolved path = " << resolved_path;
+
+    return content_blob;
+  }
+
+  mediapipe::GlCalculatorHelper gpu_helper_;
+  std::unique_ptr<face_geometry::EffectRenderer> effect_renderer_;
+};
+
+}  // namespace
+
+using FaceGeometryEffectRendererCalculator = EffectRendererCalculator;
+
+REGISTER_CALCULATOR(FaceGeometryEffectRendererCalculator);
+
+}  // namespace mediapipe

mediapipe/modules/face_geometry/effect_renderer_calculator.proto

@@ -0,0 +1,46 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator_options.proto";
+
+message FaceGeometryEffectRendererCalculatorOptions {
+  extend CalculatorOptions {
+    optional FaceGeometryEffectRendererCalculatorOptions ext = 323693808;
+  }
+
+  // Defines a path for the visual effect texture file. The effect texture is
+  // later rendered on top of the effect mesh.
+  //
+  // Please be aware about the difference between the CPU texture memory layout
+  // and the GPU texture sampler coordinate space. This renderer follows
+  // conventions discussed here: https://open.gl/textures
+  //
+  // The texture file format must be supported by the OpenCV image decoder. It
+  // must also define either an RGB or an RGBA texture.
+  optional string effect_texture_path = 1;
+
+  // Defines a path for the visual effect mesh 3D file. The effect mesh is later
+  // "attached" to the face and is driven by the face pose transformation
+  // matrix.
+  //
+  // The mesh 3D file format must be the binary `face_system.Mesh3d` proto.
+  //
+  // If is not present, the runtime face mesh will be used as the effect mesh
+  // - this mode is handy for facepaint effects.
+  optional string effect_mesh_3d_path = 2;
+}

mediapipe/modules/face_geometry/env_generator_calculator.cc

@@ -0,0 +1,81 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/modules/face_geometry/env_generator_calculator.pb.h"
+#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+
+namespace mediapipe {
+namespace {
+
+static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
+
+// A calculator that generates an environment, which describes a virtual scene.
+//
+// Output side packets:
+//   ENVIRONMENT (`face_geometry::Environment`, required)
+//     Describes an environment; includes the camera frame origin point location
+//     as well as virtual camera parameters.
+//
+// Options:
+//   environment (`face_geometry.Environment`, required):
+//     Defines an environment to be packed as the output side packet.
+//
+//     Must be valid (for details, please refer to the proto message definition
+//     comments and/or `modules/face_geometry/libs/validation_utils.h/cc`)
+//
+class EnvGeneratorCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->OutputSidePackets()
+        .Tag(kEnvironmentTag)
+        .Set<face_geometry::Environment>();
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(mediapipe::TimestampDiff(0));
+
+    const face_geometry::Environment& environment =
+        cc->Options<FaceGeometryEnvGeneratorCalculatorOptions>().environment();
+
+    MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
+        << "Invalid environment!";
+
+    cc->OutputSidePackets()
+        .Tag(kEnvironmentTag)
+        .Set(mediapipe::MakePacket<face_geometry::Environment>(environment));
+
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Process(CalculatorContext* cc) override {
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Close(CalculatorContext* cc) override {
+    return mediapipe::OkStatus();
+  }
+};
+
+}  // namespace
+
+using FaceGeometryEnvGeneratorCalculator = EnvGeneratorCalculator;
+
+REGISTER_CALCULATOR(FaceGeometryEnvGeneratorCalculator);
+
+}  // namespace mediapipe

mediapipe/modules/face_geometry/env_generator_calculator.proto

@@ -0,0 +1,32 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator_options.proto";
+import "mediapipe/modules/face_geometry/protos/environment.proto";
+
+message FaceGeometryEnvGeneratorCalculatorOptions {
+  extend CalculatorOptions {
+    optional FaceGeometryEnvGeneratorCalculatorOptions ext = 323693810;
+  }
+
+  // Defines an environment to be packed as the output side packet.
+  //
+  // Must be valid (for details, please refer to the proto message definition
+  // comments and/or `modules/face_geometry/libs/validation_utils.h/cc`)
+  optional face_geometry.Environment environment = 1;
+}

mediapipe/modules/face_geometry/face_geometry.pbtxt

@@ -0,0 +1,52 @@
+# MediaPipe graph to extract face geometry from landmarks for multiple faces.
+#
+# It is required that "geometry_pipeline_metadata.binarypb" is available at
+# "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.binarypb"
+# path during execution.
+#
+# EXAMPLE:
+#   node {
+#     calculator: "FaceGeometry"
+#     input_stream: "IMAGE_SIZE:image_size"
+#     input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
+#     input_side_packet: "ENVIRONMENT:environment"
+#     output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
+#   }
+
+type: "FaceGeometry"
+
+# The size of the input frame. The first element of the pair is the frame width;
+# the other one is the frame height.
+#
+# The face landmarks should have been detected on a frame with the same
+# ratio. If used as-is, the resulting face geometry visualization should be
+# happening on a frame with the same ratio as well.
+#
+# (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+
+# Collection of detected/predicted faces, each represented as a list of face
+# landmarks. (std::vector<NormalizedLandmarkList>)
+input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
+
+# Environment that describes the current virtual scene.
+# (face_geometry::Environment)
+input_side_packet: "ENVIRONMENT:environment"
+
+# A list of geometry data for each detected face.
+# (std::vector<face_geometry::FaceGeometry>)
+output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
+
+# Extracts face geometry for multiple faces from a vector of landmark lists.
+node {
+  calculator: "FaceGeometryPipelineCalculator"
+  input_side_packet: "ENVIRONMENT:environment"
+  input_stream: "IMAGE_SIZE:image_size"
+  input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
+  output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
+  options: {
+    [mediapipe.FaceGeometryPipelineCalculatorOptions.ext] {
+      metadata_path: "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.binarypb"
+    }
+  }
+}

mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc

@@ -0,0 +1,197 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/modules/face_geometry/geometry_pipeline_calculator.pb.h"
+#include "mediapipe/modules/face_geometry/libs/geometry_pipeline.h"
+#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
+#include "mediapipe/util/resource_util.h"
+
+namespace mediapipe {
+namespace {
+
+static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
+static constexpr char kImageSizeTag[] = "IMAGE_SIZE";
+static constexpr char kMultiFaceGeometryTag[] = "MULTI_FACE_GEOMETRY";
+static constexpr char kMultiFaceLandmarksTag[] = "MULTI_FACE_LANDMARKS";
+
+// A calculator that renders a visual effect for multiple faces.
+//
+// Inputs:
+//   IMAGE_SIZE (`std::pair<int, int>`, required):
+//     The size of the current frame. The first element of the pair is the frame
+//     width; the other one is the frame height.
+//
+//     The face landmarks should have been detected on a frame with the same
+//     ratio. If used as-is, the resulting face geometry visualization should be
+//     happening on a frame with the same ratio as well.
+//
+//   MULTI_FACE_LANDMARKS (`std::vector<NormalizedLandmarkList>`, required):
+//     A vector of face landmark lists.
+//
+// Input side packets:
+//   ENVIRONMENT (`face_geometry::Environment`, required)
+//     Describes an environment; includes the camera frame origin point location
+//     as well as virtual camera parameters.
+//
+// Output:
+//   MULTI_FACE_GEOMETRY (`std::vector<face_geometry::FaceGeometry>`, required):
+//     A vector of face geometry data.
+//
+// Options:
+//   metadata_path (`string`, optional):
+//     Defines a path for the geometry pipeline metadata file.
+//
+//     The geometry pipeline metadata file format must be the binary
+//     `face_geometry.GeometryPipelineMetadata` proto.
+//
+class GeometryPipelineCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->InputSidePackets()
+        .Tag(kEnvironmentTag)
+        .Set<face_geometry::Environment>();
+    cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
+    cc->Inputs()
+        .Tag(kMultiFaceLandmarksTag)
+        .Set<std::vector<NormalizedLandmarkList>>();
+    cc->Outputs()
+        .Tag(kMultiFaceGeometryTag)
+        .Set<std::vector<face_geometry::FaceGeometry>>();
+
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(mediapipe::TimestampDiff(0));
+
+    const auto& options = cc->Options<FaceGeometryPipelineCalculatorOptions>();
+
+    ASSIGN_OR_RETURN(
+        face_geometry::GeometryPipelineMetadata metadata,
+        ReadMetadataFromFile(options.metadata_path()),
+        _ << "Failed to read the geometry pipeline metadata from file!");
+
+    MP_RETURN_IF_ERROR(
+        face_geometry::ValidateGeometryPipelineMetadata(metadata))
+        << "Invalid geometry pipeline metadata!";
+
+    const face_geometry::Environment& environment =
+        cc->InputSidePackets()
+            .Tag(kEnvironmentTag)
+            .Get<face_geometry::Environment>();
+
+    MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
+        << "Invalid environment!";
+
+    ASSIGN_OR_RETURN(
+        geometry_pipeline_,
+        face_geometry::CreateGeometryPipeline(environment, metadata),
+        _ << "Failed to create a geometry pipeline!");
+
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Process(CalculatorContext* cc) override {
+    // Both the `IMAGE_SIZE` and the `MULTI_FACE_LANDMARKS` streams are required
+    // to have a non-empty packet. In case this requirement is not met, there's
+    // nothing to be processed at the current timestamp.
+    if (cc->Inputs().Tag(kImageSizeTag).IsEmpty() ||
+        cc->Inputs().Tag(kMultiFaceLandmarksTag).IsEmpty()) {
+      return mediapipe::OkStatus();
+    }
+
+    const auto& image_size =
+        cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
+    const auto& multi_face_landmarks =
+        cc->Inputs()
+            .Tag(kMultiFaceLandmarksTag)
+            .Get<std::vector<NormalizedLandmarkList>>();
+
+    auto multi_face_geometry =
+        absl::make_unique<std::vector<face_geometry::FaceGeometry>>();
+
+    ASSIGN_OR_RETURN(
+        *multi_face_geometry,
+        geometry_pipeline_->EstimateFaceGeometry(
+            multi_face_landmarks,  //
+            /*frame_width*/ image_size.first,
+            /*frame_height*/ image_size.second),
+        _ << "Failed to estimate face geometry for multiple faces!");
+
+    cc->Outputs()
+        .Tag(kMultiFaceGeometryTag)
+        .AddPacket(mediapipe::Adopt<std::vector<face_geometry::FaceGeometry>>(
+                       multi_face_geometry.release())
+                       .At(cc->InputTimestamp()));
+
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Close(CalculatorContext* cc) override {
+    return mediapipe::OkStatus();
+  }
+
+ private:
+  static mediapipe::StatusOr<face_geometry::GeometryPipelineMetadata>
+  ReadMetadataFromFile(const std::string& metadata_path) {
+    ASSIGN_OR_RETURN(std::string metadata_blob,
+                     ReadContentBlobFromFile(metadata_path),
+                     _ << "Failed to read a metadata blob from file!");
+
+    face_geometry::GeometryPipelineMetadata metadata;
+    RET_CHECK(metadata.ParseFromString(metadata_blob))
+        << "Failed to parse a metadata proto from a binary blob!";
+
+    return metadata;
+  }
+
+  static mediapipe::StatusOr<std::string> ReadContentBlobFromFile(
+      const std::string& unresolved_path) {
+    ASSIGN_OR_RETURN(std::string resolved_path,
+                     mediapipe::PathToResourceAsFile(unresolved_path),
+                     _ << "Failed to resolve path! Path = " << unresolved_path);
+
+    std::string content_blob;
+    MP_RETURN_IF_ERROR(
+        mediapipe::GetResourceContents(resolved_path, &content_blob))
+        << "Failed to read content blob! Resolved path = " << resolved_path;
+
+    return content_blob;
+  }
+
+  std::unique_ptr<face_geometry::GeometryPipeline> geometry_pipeline_;
+};
+
+}  // namespace
+
+using FaceGeometryPipelineCalculator = GeometryPipelineCalculator;
+
+REGISTER_CALCULATOR(FaceGeometryPipelineCalculator);
+
+}  // namespace mediapipe

mediapipe/modules/face_geometry/geometry_pipeline_calculator.proto

@@ -0,0 +1,27 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator_options.proto";
+
+message FaceGeometryPipelineCalculatorOptions {
+  extend CalculatorOptions {
+    optional FaceGeometryPipelineCalculatorOptions ext = 323693812;
+  }
+
+  optional string metadata_path = 1;
+}

mediapipe/modules/face_geometry/libs/BUILD

@@ -0,0 +1,103 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "effect_renderer",
+    srcs = ["effect_renderer.cc"],
+    hdrs = ["effect_renderer.h"],
+    deps = [
+        ":mesh_3d_utils",
+        ":validation_utils",
+        "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:matrix_data_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "//mediapipe/gpu:gl_base",
+        "//mediapipe/gpu:shader_util",
+        "//mediapipe/modules/face_geometry/protos:environment_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
+
+cc_library(
+    name = "geometry_pipeline",
+    srcs = ["geometry_pipeline.cc"],
+    hdrs = ["geometry_pipeline.h"],
+    deps = [
+        ":mesh_3d_utils",
+        ":procrustes_solver",
+        ":validation_utils",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:matrix_data_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "//mediapipe/modules/face_geometry/protos:environment_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
+        "@com_google_absl//absl/memory",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_library(
+    name = "mesh_3d_utils",
+    srcs = ["mesh_3d_utils.cc"],
+    hdrs = ["mesh_3d_utils.h"],
+    deps = [
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:statusor",
+        "//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
+    ],
+)
+
+cc_library(
+    name = "procrustes_solver",
+    srcs = ["procrustes_solver.cc"],
+    hdrs = ["procrustes_solver.h"],
+    deps = [
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "@com_google_absl//absl/memory",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_library(
+    name = "validation_utils",
+    srcs = ["validation_utils.cc"],
+    hdrs = ["validation_utils.h"],
+    deps = [
+        ":mesh_3d_utils",
+        "//mediapipe/framework/formats:matrix_data_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/modules/face_geometry/protos:environment_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
+    ],
+)

mediapipe/modules/face_geometry/libs/effect_renderer.cc

@@ -0,0 +1,734 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/face_geometry/libs/effect_renderer.h"
+
+#include <cmath>
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/types/optional.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/matrix_data.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/gpu/gl_base.h"
+#include "mediapipe/gpu/shader_util.h"
+#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
+#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
+
+namespace mediapipe::face_geometry {
+namespace {
+
+struct RenderableMesh3d {
+  static mediapipe::StatusOr<RenderableMesh3d> CreateFromProtoMesh3d(
+      const Mesh3d& proto_mesh_3d) {
+    Mesh3d::VertexType vertex_type = proto_mesh_3d.vertex_type();
+
+    RenderableMesh3d renderable_mesh_3d;
+    renderable_mesh_3d.vertex_size = GetVertexSize(vertex_type);
+    ASSIGN_OR_RETURN(
+        renderable_mesh_3d.vertex_position_size,
+        GetVertexComponentSize(vertex_type, VertexComponent::POSITION),
+        _ << "Failed to get the position vertex size!");
+    ASSIGN_OR_RETURN(
+        renderable_mesh_3d.tex_coord_position_size,
+        GetVertexComponentSize(vertex_type, VertexComponent::TEX_COORD),
+        _ << "Failed to get the tex coord vertex size!");
+    ASSIGN_OR_RETURN(
+        renderable_mesh_3d.vertex_position_offset,
+        GetVertexComponentOffset(vertex_type, VertexComponent::POSITION),
+        _ << "Failed to get the position vertex offset!");
+    ASSIGN_OR_RETURN(
+        renderable_mesh_3d.tex_coord_position_offset,
+        GetVertexComponentOffset(vertex_type, VertexComponent::TEX_COORD),
+        _ << "Failed to get the tex coord vertex offset!");
+
+    switch (proto_mesh_3d.primitive_type()) {
+      case Mesh3d::TRIANGLE:
+        renderable_mesh_3d.primitive_type = GL_TRIANGLES;
+        break;
+
+      default:
+        RET_CHECK_FAIL() << "Only triangle primitive types are supported!";
+    }
+
+    renderable_mesh_3d.vertex_buffer.reserve(
+        proto_mesh_3d.vertex_buffer_size());
+    for (float vertex_element : proto_mesh_3d.vertex_buffer()) {
+      renderable_mesh_3d.vertex_buffer.push_back(vertex_element);
+    }
+
+    renderable_mesh_3d.index_buffer.reserve(proto_mesh_3d.index_buffer_size());
+    for (uint32_t index_element : proto_mesh_3d.index_buffer()) {
+      RET_CHECK_LE(index_element, std::numeric_limits<uint16_t>::max())
+          << "Index buffer elements must fit into the `uint16` type in order "
+             "to be renderable!";
+
+      renderable_mesh_3d.index_buffer.push_back(
+          static_cast<uint16_t>(index_element));
+    }
+
+    return renderable_mesh_3d;
+  }
+
+  uint32_t vertex_size;
+  uint32_t vertex_position_size;
+  uint32_t tex_coord_position_size;
+  uint32_t vertex_position_offset;
+  uint32_t tex_coord_position_offset;
+  uint32_t primitive_type;
+
+  std::vector<float> vertex_buffer;
+  std::vector<uint16_t> index_buffer;
+};
+
+class Texture {
+ public:
+  static mediapipe::StatusOr<std::unique_ptr<Texture>> WrapExternalTexture(
+      GLuint handle, GLenum target, int width, int height) {
+    RET_CHECK(handle) << "External texture must have a non-null handle!";
+    return absl::WrapUnique(new Texture(handle, target, width, height,
+                                        /*is_owned*/ false));
+  }
+
+  static mediapipe::StatusOr<std::unique_ptr<Texture>> CreateFromImageFrame(
+      const ImageFrame& image_frame) {
+    RET_CHECK(image_frame.IsAligned(ImageFrame::kGlDefaultAlignmentBoundary))
+        << "Image frame memory must be aligned for GL usage!";
+
+    RET_CHECK(image_frame.Width() > 0 && image_frame.Height() > 0)
+        << "Image frame must have positive dimensions!";
+
+    RET_CHECK(image_frame.Format() == ImageFormat::SRGB ||
+              image_frame.Format() == ImageFormat::SRGBA)
+        << "Image frame format must be either SRGB or SRGBA!";
+
+    GLint image_format;
+    switch (image_frame.NumberOfChannels()) {
+      case 3:
+        image_format = GL_RGB;
+        break;
+      case 4:
+        image_format = GL_RGBA;
+        break;
+      default:
+        RET_CHECK_FAIL()
+            << "Unexpected number of channels; expected 3 or 4, got "
+            << image_frame.NumberOfChannels() << "!";
+    }
+
+    GLuint handle;
+    glGenTextures(1, &handle);
+    RET_CHECK(handle) << "Failed to initialize an OpenGL texture!";
+
+    glBindTexture(GL_TEXTURE_2D, handle);
+    glTexParameteri(GL_TEXTURE_2D, GL_NEAREST_MIPMAP_LINEAR, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    glTexImage2D(GL_TEXTURE_2D, 0, image_format, image_frame.Width(),
+                 image_frame.Height(), 0, image_format, GL_UNSIGNED_BYTE,
+                 image_frame.PixelData());
+    glGenerateMipmap(GL_TEXTURE_2D);
+    glBindTexture(GL_TEXTURE_2D, 0);
+
+    return absl::WrapUnique(new Texture(
+        handle, GL_TEXTURE_2D, image_frame.Width(), image_frame.Height(),
+        /*is_owned*/ true));
+  }
+
+  ~Texture() {
+    if (is_owned_) {
+      glDeleteProgram(handle_);
+    }
+  }
+
+  GLuint handle() const { return handle_; }
+  GLenum target() const { return target_; }
+  int width() const { return width_; }
+  int height() const { return height_; }
+
+ private:
+  Texture(GLuint handle, GLenum target, int width, int height, bool is_owned)
+      : handle_(handle),
+        target_(target),
+        width_(width),
+        height_(height),
+        is_owned_(is_owned) {}
+
+  GLuint handle_;
+  GLenum target_;
+  int width_;
+  int height_;
+  bool is_owned_;
+};
+
+class RenderTarget {
+ public:
+  static mediapipe::StatusOr<std::unique_ptr<RenderTarget>> Create() {
+    GLuint framebuffer_handle;
+    glGenFramebuffers(1, &framebuffer_handle);
+    RET_CHECK(framebuffer_handle)
+        << "Failed to initialize an OpenGL framebuffer!";
+
+    return absl::WrapUnique(new RenderTarget(framebuffer_handle));
+  }
+
+  ~RenderTarget() {
+    glDeleteFramebuffers(1, &framebuffer_handle_);
+    // Renderbuffer handle might have never been created if this render target
+    // is destroyed before `SetColorbuffer()` is called for the first time.
+    if (renderbuffer_handle_) {
+      glDeleteFramebuffers(1, &renderbuffer_handle_);
+    }
+  }
+
+  mediapipe::Status SetColorbuffer(const Texture& colorbuffer_texture) {
+    glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_handle_);
+    glViewport(0, 0, colorbuffer_texture.width(), colorbuffer_texture.height());
+
+    glActiveTexture(GL_TEXTURE0);
+    glBindTexture(colorbuffer_texture.target(), colorbuffer_texture.handle());
+    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
+                           colorbuffer_texture.target(),
+                           colorbuffer_texture.handle(),
+                           /*level*/ 0);
+    glBindTexture(colorbuffer_texture.target(), 0);
+
+    // If the existing depth buffer has different dimensions, delete it.
+    if (renderbuffer_handle_ &&
+        (viewport_width_ != colorbuffer_texture.width() ||
+         viewport_height_ != colorbuffer_texture.height())) {
+      glDeleteRenderbuffers(1, &renderbuffer_handle_);
+      renderbuffer_handle_ = 0;
+    }
+
+    // If there is no depth buffer, create one.
+    if (!renderbuffer_handle_) {
+      glGenRenderbuffers(1, &renderbuffer_handle_);
+      RET_CHECK(renderbuffer_handle_)
+          << "Failed to initialize an OpenGL renderbuffer!";
+      glBindRenderbuffer(GL_RENDERBUFFER, renderbuffer_handle_);
+      glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT16,
+                            colorbuffer_texture.width(),
+                            colorbuffer_texture.height());
+      glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
+                                GL_RENDERBUFFER, renderbuffer_handle_);
+      glBindRenderbuffer(GL_RENDERBUFFER, 0);
+    }
+
+    viewport_width_ = colorbuffer_texture.width();
+    viewport_height_ = colorbuffer_texture.height();
+
+    glBindFramebuffer(GL_FRAMEBUFFER, 0);
+    glFlush();
+
+    return mediapipe::OkStatus();
+  }
+
+  void Bind() const {
+    glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_handle_);
+    glViewport(0, 0, viewport_width_, viewport_height_);
+  }
+
+  void Unbind() const { glBindFramebuffer(GL_FRAMEBUFFER, 0); }
+
+  void Clear() const {
+    Bind();
+    glEnable(GL_DEPTH_TEST);
+    glDepthMask(GL_TRUE);
+
+    glClearColor(0.f, 0.f, 0.f, 0.f);
+    glClearDepthf(1.f);
+    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+
+    glDepthMask(GL_FALSE);
+    glDisable(GL_DEPTH_TEST);
+
+    Unbind();
+    glFlush();
+  }
+
+ private:
+  explicit RenderTarget(GLuint framebuffer_handle)
+      : framebuffer_handle_(framebuffer_handle),
+        renderbuffer_handle_(0),
+        viewport_width_(-1),
+        viewport_height_(-1) {}
+
+  GLuint framebuffer_handle_;
+  GLuint renderbuffer_handle_;
+  int viewport_width_;
+  int viewport_height_;
+};
+
+class Renderer {
+ public:
+  enum class RenderMode { OPAQUE, OVERDRAW, OCCLUSION };
+
+  static mediapipe::StatusOr<std::unique_ptr<Renderer>> Create() {
+    static const GLint kAttrLocation[NUM_ATTRIBUTES] = {
+        ATTRIB_VERTEX,
+        ATTRIB_TEXTURE_POSITION,
+    };
+    static const GLchar* kAttrName[NUM_ATTRIBUTES] = {
+        "position",
+        "tex_coord",
+    };
+
+    static const GLchar* kVertSrc = R"(
+      uniform mat4 projection_mat;
+      uniform mat4 model_mat;
+
+      attribute vec4 position;
+      attribute vec4 tex_coord;
+
+      varying vec2 v_tex_coord;
+
+      void main() {
+        v_tex_coord = tex_coord.xy;
+        gl_Position = projection_mat * model_mat * position;
+      }
+    )";
+
+    static const GLchar* kFragSrc = R"(
+      precision mediump float;
+
+      varying vec2 v_tex_coord;
+      uniform sampler2D texture;
+
+      void main() {
+        gl_FragColor = texture2D(texture, v_tex_coord);
+      }
+    )";
+
+    GLuint program_handle = 0;
+    GlhCreateProgram(kVertSrc, kFragSrc, NUM_ATTRIBUTES,
+                     (const GLchar**)&kAttrName[0], kAttrLocation,
+                     &program_handle);
+    RET_CHECK(program_handle) << "Problem initializing the texture program!";
+    GLint projection_mat_uniform =
+        glGetUniformLocation(program_handle, "projection_mat");
+    GLint model_mat_uniform = glGetUniformLocation(program_handle, "model_mat");
+    GLint texture_uniform = glGetUniformLocation(program_handle, "texture");
+
+    RET_CHECK_NE(projection_mat_uniform, -1)
+        << "Failed to find `projection_mat` uniform!";
+    RET_CHECK_NE(model_mat_uniform, -1)
+        << "Failed to find `model_mat` uniform!";
+    RET_CHECK_NE(texture_uniform, -1) << "Failed to find `texture` uniform!";
+
+    return absl::WrapUnique(new Renderer(program_handle, projection_mat_uniform,
+                                         model_mat_uniform, texture_uniform));
+  }
+
+  ~Renderer() { glDeleteProgram(program_handle_); }
+
+  mediapipe::Status Render(const RenderTarget& render_target,
+                           const Texture& texture,
+                           const RenderableMesh3d& mesh_3d,
+                           const std::array<float, 16>& projection_mat,
+                           const std::array<float, 16>& model_mat,
+                           RenderMode render_mode) const {
+    glUseProgram(program_handle_);
+    // Set up the GL state.
+    glEnable(GL_BLEND);
+    glFrontFace(GL_CCW);
+    switch (render_mode) {
+      case RenderMode::OPAQUE:
+        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+        glEnable(GL_DEPTH_TEST);
+        glDepthMask(GL_TRUE);
+        break;
+
+      case RenderMode::OVERDRAW:
+        glBlendFunc(GL_ONE, GL_ZERO);
+        glDisable(GL_DEPTH_TEST);
+        glDepthMask(GL_FALSE);
+        break;
+
+      case RenderMode::OCCLUSION:
+        glBlendFunc(GL_ZERO, GL_ONE);
+        glEnable(GL_DEPTH_TEST);
+        glDepthMask(GL_TRUE);
+        break;
+    }
+
+    render_target.Bind();
+    // Set up vertex attributes.
+    glVertexAttribPointer(
+        ATTRIB_VERTEX, mesh_3d.vertex_position_size, GL_FLOAT, 0,
+        mesh_3d.vertex_size * sizeof(float),
+        mesh_3d.vertex_buffer.data() + mesh_3d.vertex_position_offset);
+    glEnableVertexAttribArray(ATTRIB_VERTEX);
+    glVertexAttribPointer(
+        ATTRIB_TEXTURE_POSITION, mesh_3d.tex_coord_position_size, GL_FLOAT, 0,
+        mesh_3d.vertex_size * sizeof(float),
+        mesh_3d.vertex_buffer.data() + mesh_3d.tex_coord_position_offset);
+    glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
+    // Set up textures and uniforms.
+    glActiveTexture(GL_TEXTURE1);
+    glBindTexture(texture.target(), texture.handle());
+    glUniform1i(texture_uniform_, 1);
+    glUniformMatrix4fv(projection_mat_uniform_, 1, GL_FALSE,
+                       projection_mat.data());
+    glUniformMatrix4fv(model_mat_uniform_, 1, GL_FALSE, model_mat.data());
+    // Draw the mesh.
+    glDrawElements(mesh_3d.primitive_type, mesh_3d.index_buffer.size(),
+                   GL_UNSIGNED_SHORT, mesh_3d.index_buffer.data());
+    // Unbind textures and uniforms.
+    glActiveTexture(GL_TEXTURE1);
+    glBindTexture(texture.target(), 0);
+    render_target.Unbind();
+    // Unbind vertex attributes.
+    glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
+    glDisableVertexAttribArray(ATTRIB_VERTEX);
+    // Restore the GL state.
+    glDepthMask(GL_FALSE);
+    glDisable(GL_DEPTH_TEST);
+    glDisable(GL_BLEND);
+
+    glUseProgram(0);
+    glFlush();
+
+    return mediapipe::OkStatus();
+  }
+
+ private:
+  enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
+
+  Renderer(GLuint program_handle, GLint projection_mat_uniform,
+           GLint model_mat_uniform, GLint texture_uniform)
+      : program_handle_(program_handle),
+        projection_mat_uniform_(projection_mat_uniform),
+        model_mat_uniform_(model_mat_uniform),
+        texture_uniform_(texture_uniform) {}
+
+  GLuint program_handle_;
+  GLint projection_mat_uniform_;
+  GLint model_mat_uniform_;
+  GLint texture_uniform_;
+};
+
+class EffectRendererImpl : public EffectRenderer {
+ public:
+  EffectRendererImpl(
+      const Environment& environment,
+      std::unique_ptr<RenderTarget> render_target,
+      std::unique_ptr<Renderer> renderer,
+      RenderableMesh3d&& renderable_quad_mesh_3d,
+      absl::optional<RenderableMesh3d>&& renderable_effect_mesh_3d,
+      std::unique_ptr<Texture> empty_color_texture,
+      std::unique_ptr<Texture> effect_texture)
+      : environment_(environment),
+        render_target_(std::move(render_target)),
+        renderer_(std::move(renderer)),
+        renderable_quad_mesh_3d_(std::move(renderable_quad_mesh_3d)),
+        renderable_effect_mesh_3d_(std::move(renderable_effect_mesh_3d)),
+        empty_color_texture_(std::move(empty_color_texture)),
+        effect_texture_(std::move(effect_texture)),
+        identity_matrix_(Create4x4IdentityMatrix()) {}
+
+  mediapipe::Status RenderEffect(
+      const std::vector<FaceGeometry>& multi_face_geometry,
+      int frame_width,            //
+      int frame_height,           //
+      GLenum src_texture_target,  //
+      GLuint src_texture_name,    //
+      GLenum dst_texture_target,  //
+      GLuint dst_texture_name) {
+    // Validate input arguments.
+    MP_RETURN_IF_ERROR(ValidateFrameDimensions(frame_width, frame_height))
+        << "Invalid frame dimensions!";
+    RET_CHECK(src_texture_name > 0 && dst_texture_name > 0)
+        << "Both source and destination texture names must be non-null!";
+    RET_CHECK_NE(src_texture_name, dst_texture_name)
+        << "Source and destination texture names must be different!";
+
+    // Validate all input face geometries.
+    for (const FaceGeometry& face_geometry : multi_face_geometry) {
+      MP_RETURN_IF_ERROR(ValidateFaceGeometry(face_geometry))
+          << "Invalid face geometry!";
+    }
+
+    // Wrap both source and destination textures.
+    ASSIGN_OR_RETURN(
+        std::unique_ptr<Texture> src_texture,
+        Texture::WrapExternalTexture(src_texture_name, src_texture_target,
+                                     frame_width, frame_height),
+        _ << "Failed to wrap the external source texture");
+    ASSIGN_OR_RETURN(
+        std::unique_ptr<Texture> dst_texture,
+        Texture::WrapExternalTexture(dst_texture_name, dst_texture_target,
+                                     frame_width, frame_height),
+        _ << "Failed to wrap the external destination texture");
+
+    // Set the destination texture as the color buffer. Then, clear both the
+    // color and the depth buffers for the render target.
+    MP_RETURN_IF_ERROR(render_target_->SetColorbuffer(*dst_texture))
+        << "Failed to set the destination texture as the colorbuffer!";
+    render_target_->Clear();
+
+    // Render the source texture on top of the quad mesh (i.e. make a copy)
+    // into the render target.
+    MP_RETURN_IF_ERROR(renderer_->Render(
+        *render_target_, *src_texture, renderable_quad_mesh_3d_,
+        identity_matrix_, identity_matrix_, Renderer::RenderMode::OVERDRAW))
+        << "Failed to render the source texture on top of the quad mesh!";
+
+    // Extract pose transform matrices and meshes from the face geometry data;
+    const int num_faces = multi_face_geometry.size();
+
+    std::vector<std::array<float, 16>> face_pose_transform_matrices(num_faces);
+    std::vector<RenderableMesh3d> renderable_face_meshes(num_faces);
+    for (int i = 0; i < num_faces; ++i) {
+      const FaceGeometry& face_geometry = multi_face_geometry[i];
+
+      // Extract the face pose transformation matrix.
+      ASSIGN_OR_RETURN(
+          face_pose_transform_matrices[i],
+          Convert4x4MatrixDataToArrayFormat(
+              face_geometry.pose_transform_matrix()),
+          _ << "Failed to extract the face pose transformation matrix!");
+
+      // Extract the face mesh as a renderable.
+      ASSIGN_OR_RETURN(
+          renderable_face_meshes[i],
+          RenderableMesh3d::CreateFromProtoMesh3d(face_geometry.mesh()),
+          _ << "Failed to extract a renderable face mesh!");
+    }
+
+    // Create a perspective matrix using the frame aspect ratio.
+    std::array<float, 16> perspective_matrix = CreatePerspectiveMatrix(
+        /*aspect_ratio*/ static_cast<float>(frame_width) / frame_height);
+
+    // Render a face mesh occluder for each face.
+    for (int i = 0; i < num_faces; ++i) {
+      const std::array<float, 16>& face_pose_transform_matrix =
+          face_pose_transform_matrices[i];
+      const RenderableMesh3d& renderable_face_mesh = renderable_face_meshes[i];
+
+      // Render the face mesh using the empty color texture, i.e. the face
+      // mesh occluder.
+      //
+      // For occlusion, the pose transformation is moved ~1mm away from camera
+      // in order to allow the face mesh texture to be rendered without
+      // failing the depth test.
+      std::array<float, 16> occlusion_face_pose_transform_matrix =
+          face_pose_transform_matrix;
+      occlusion_face_pose_transform_matrix[14] -= 0.1f;  // ~ 1mm
+      MP_RETURN_IF_ERROR(renderer_->Render(
+          *render_target_, *empty_color_texture_, renderable_face_mesh,
+          perspective_matrix, occlusion_face_pose_transform_matrix,
+          Renderer::RenderMode::OCCLUSION))
+          << "Failed to render the face mesh occluder!";
+    }
+
+    // Render the main face mesh effect component for each face.
+    for (int i = 0; i < num_faces; ++i) {
+      const std::array<float, 16>& face_pose_transform_matrix =
+          face_pose_transform_matrices[i];
+
+      // If there is no effect 3D mesh provided, then the face mesh itself is
+      // used as a topology for rendering (for example, this can be used for
+      // facepaint effects or AR makeup).
+      const RenderableMesh3d& main_effect_mesh_3d =
+          renderable_effect_mesh_3d_ ? *renderable_effect_mesh_3d_
+                                     : renderable_face_meshes[i];
+
+      MP_RETURN_IF_ERROR(renderer_->Render(
+          *render_target_, *effect_texture_, main_effect_mesh_3d,
+          perspective_matrix, face_pose_transform_matrix,
+          Renderer::RenderMode::OPAQUE))
+          << "Failed to render the main effect pass!";
+    }
+
+    // At this point in the code, the destination texture must contain the
+    // correctly renderer effect, so we should just return.
+    return mediapipe::OkStatus();
+  }
+
+ private:
+  std::array<float, 16> CreatePerspectiveMatrix(float aspect_ratio) const {
+    static constexpr float kDegreesToRadians = M_PI / 180.f;
+
+    std::array<float, 16> perspective_matrix;
+    perspective_matrix.fill(0.f);
+
+    const auto& env_camera = environment_.perspective_camera();
+    // Standard perspective projection matrix calculations.
+    const float f = 1.0f / std::tan(kDegreesToRadians *
+                                    env_camera.vertical_fov_degrees() / 2.f);
+
+    const float denom = 1.0f / (env_camera.near() - env_camera.far());
+    perspective_matrix[0] = f / aspect_ratio;
+    perspective_matrix[5] = f;
+    perspective_matrix[10] = (env_camera.near() + env_camera.far()) * denom;
+    perspective_matrix[11] = -1.f;
+    perspective_matrix[14] = 2.f * env_camera.far() * env_camera.near() * denom;
+
+    // If the environment's origin point location is in the top left corner,
+    // then skip additional flip along Y-axis is required to render correctly.
+    if (environment_.origin_point_location() ==
+        OriginPointLocation::TOP_LEFT_CORNER) {
+      perspective_matrix[5] *= -1.f;
+    }
+
+    return perspective_matrix;
+  }
+
+  static std::array<float, 16> Create4x4IdentityMatrix() {
+    return {1.f, 0.f, 0.f, 0.f,  //
+            0.f, 1.f, 0.f, 0.f,  //
+            0.f, 0.f, 1.f, 0.f,  //
+            0.f, 0.f, 0.f, 1.f};
+  }
+
+  static mediapipe::StatusOr<std::array<float, 16>>
+  Convert4x4MatrixDataToArrayFormat(const MatrixData& matrix_data) {
+    RET_CHECK(matrix_data.rows() == 4 &&  //
+              matrix_data.cols() == 4 &&  //
+              matrix_data.packed_data_size() == 16)
+        << "The matrix data must define a 4x4 matrix!";
+
+    std::array<float, 16> matrix_array;
+    for (int i = 0; i < 16; i++) {
+      matrix_array[i] = matrix_data.packed_data(i);
+    }
+
+    // Matrix array must be in the OpenGL-friendly column-major order. If
+    // `matrix_data` is in the row-major order, then transpose.
+    if (matrix_data.layout() == MatrixData::ROW_MAJOR) {
+      std::swap(matrix_array[1], matrix_array[4]);
+      std::swap(matrix_array[2], matrix_array[8]);
+      std::swap(matrix_array[3], matrix_array[12]);
+      std::swap(matrix_array[6], matrix_array[9]);
+      std::swap(matrix_array[7], matrix_array[13]);
+      std::swap(matrix_array[11], matrix_array[14]);
+    }
+
+    return matrix_array;
+  }
+
+  Environment environment_;
+
+  std::unique_ptr<RenderTarget> render_target_;
+  std::unique_ptr<Renderer> renderer_;
+
+  RenderableMesh3d renderable_quad_mesh_3d_;
+  absl::optional<RenderableMesh3d> renderable_effect_mesh_3d_;
+
+  std::unique_ptr<Texture> empty_color_texture_;
+  std::unique_ptr<Texture> effect_texture_;
+
+  std::array<float, 16> identity_matrix_;
+};
+
+Mesh3d CreateQuadMesh3d() {
+  static constexpr float kQuadMesh3dVertexBuffer[] = {
+      -1.f, -1.f, 0.f, 0.f, 0.f,  //
+      1.f,  -1.f, 0.f, 1.f, 0.f,  //
+      -1.f, 1.f,  0.f, 0.f, 1.f,  //
+      1.f,  1.f,  0.f, 1.f, 1.f,  //
+  };
+  static constexpr uint16_t kQuadMesh3dIndexBuffer[] = {0, 1, 2, 1, 3, 2};
+
+  static constexpr int kQuadMesh3dVertexBufferSize =
+      sizeof(kQuadMesh3dVertexBuffer) / sizeof(float);
+  static constexpr int kQuadMesh3dIndexBufferSize =
+      sizeof(kQuadMesh3dIndexBuffer) / sizeof(uint16_t);
+
+  Mesh3d quad_mesh_3d;
+  quad_mesh_3d.set_vertex_type(Mesh3d::VERTEX_PT);
+  quad_mesh_3d.set_primitive_type(Mesh3d::TRIANGLE);
+  for (int i = 0; i < kQuadMesh3dVertexBufferSize; ++i) {
+    quad_mesh_3d.add_vertex_buffer(kQuadMesh3dVertexBuffer[i]);
+  }
+  for (int i = 0; i < kQuadMesh3dIndexBufferSize; ++i) {
+    quad_mesh_3d.add_index_buffer(kQuadMesh3dIndexBuffer[i]);
+  }
+
+  return quad_mesh_3d;
+}
+
+ImageFrame CreateEmptyColorTexture() {
+  static constexpr ImageFormat::Format kEmptyColorTextureFormat =
+      ImageFormat::SRGBA;
+  static constexpr int kEmptyColorTextureWidth = 1;
+  static constexpr int kEmptyColorTextureHeight = 1;
+
+  ImageFrame empty_color_texture(
+      kEmptyColorTextureFormat, kEmptyColorTextureWidth,
+      kEmptyColorTextureHeight, ImageFrame::kGlDefaultAlignmentBoundary);
+  empty_color_texture.SetToZero();
+
+  return empty_color_texture;
+}
+
+}  // namespace
+
+mediapipe::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
+    const Environment& environment,                //
+    const absl::optional<Mesh3d>& effect_mesh_3d,  //
+    ImageFrame&& effect_texture) {
+  MP_RETURN_IF_ERROR(ValidateEnvironment(environment))
+      << "Invalid environment!";
+  if (effect_mesh_3d) {
+    MP_RETURN_IF_ERROR(ValidateMesh3d(*effect_mesh_3d))
+        << "Invalid effect 3D mesh!";
+  }
+
+  ASSIGN_OR_RETURN(std::unique_ptr<RenderTarget> render_target,
+                   RenderTarget::Create(),
+                   _ << "Failed to create a render target!");
+  ASSIGN_OR_RETURN(std::unique_ptr<Renderer> renderer, Renderer::Create(),
+                   _ << "Failed to create a renderer!");
+  ASSIGN_OR_RETURN(RenderableMesh3d renderable_quad_mesh_3d,
+                   RenderableMesh3d::CreateFromProtoMesh3d(CreateQuadMesh3d()),
+                   _ << "Failed to create a renderable quad mesh!");
+  absl::optional<RenderableMesh3d> renderable_effect_mesh_3d;
+  if (effect_mesh_3d) {
+    ASSIGN_OR_RETURN(renderable_effect_mesh_3d,
+                     RenderableMesh3d::CreateFromProtoMesh3d(*effect_mesh_3d),
+                     _ << "Failed to create a renderable effect mesh!");
+  }
+  ASSIGN_OR_RETURN(std::unique_ptr<Texture> empty_color_gl_texture,
+                   Texture::CreateFromImageFrame(CreateEmptyColorTexture()),
+                   _ << "Failed to create an empty color texture!");
+  ASSIGN_OR_RETURN(std::unique_ptr<Texture> effect_gl_texture,
+                   Texture::CreateFromImageFrame(effect_texture),
+                   _ << "Failed to create an effect texture!");
+
+  std::unique_ptr<EffectRenderer> result =
+      absl::make_unique<EffectRendererImpl>(
+          environment, std::move(render_target), std::move(renderer),
+          std::move(renderable_quad_mesh_3d),
+          std::move(renderable_effect_mesh_3d),
+          std::move(empty_color_gl_texture), std::move(effect_gl_texture));
+
+  return result;
+}
+
+}  // namespace mediapipe::face_geometry

mediapipe/modules/face_geometry/libs/effect_renderer.h

@@ -0,0 +1,92 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_MODULES_FACE_GEOMETRY_LIBS_EFFECT_RENDERER_H_
+#define MEDIAPIPE_MODULES_FACE_GEOMETRY_LIBS_EFFECT_RENDERER_H_
+
+#include <memory>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/gpu/gl_base.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
+
+namespace mediapipe::face_geometry {
+
+// Encapsulates a stateful face effect renderer.
+class EffectRenderer {
+ public:
+  virtual ~EffectRenderer() = default;
+
+  // Renders a face effect based on the multiple facial geometries.
+  //
+  // Must be called in the same GL context as was used upon initialization.
+  //
+  // Each of the `multi_face_geometry` must be valid (for details, please refer
+  // to the proto message definition comments and/or `validation_utils.h/cc`).
+  // Additionally, all face mesh index buffer elements must fit into the
+  // `uint16` type in order to be renderable.
+  //
+  // Both `frame_width` and `frame_height` must be positive.
+  //
+  // Both `src_texture_name` and `dst_texture_name` must be positive and
+  // reference existing OpenGL textures in the current context. They should also
+  // reference different textures as the in-place effect rendering is not yet
+  // supported.
+  virtual mediapipe::Status RenderEffect(
+      const std::vector<FaceGeometry>& multi_face_geometry,
+      int frame_width,            //
+      int frame_height,           //
+      GLenum src_texture_target,  //
+      GLuint src_texture_name,    //
+      GLenum dst_texture_target,  //
+      GLuint dst_texture_name) = 0;
+};
+
+// Creates an instance of `EffectRenderer`.
+//
+// `effect_mesh_3d` defines a rigid 3d mesh which is "attached" to the face and
+// is driven by the face pose transformation matrix. If is not present, the
+// runtime face mesh will be used as the effect mesh - this mode is handy for
+// facepaint effects. In both rendering modes, the face mesh is first rendered
+// as an occluder straight into the depth buffer. This step helps to create a
+// more believable effect via hiding invisible elements behind the face surface.
+//
+// `effect_texture` defines the color texture to be rendered on top of the
+// effect mesh. Please be aware about the difference between the CPU texture
+// memory layout and the GPU texture sampler coordinate space. This renderer
+// follows conventions discussed here: https://open.gl/textures
+//
+// Must be called in the same GL context as will be used for rendering.
+//
+// Both `environment` and `effect_mesh_3d` (is present) must be valid (for
+// details, please refer to the proto message definition comments and/or
+// `validation_utils.h/cc`). Additionally, `effect_mesh_3d`s index buffer
+// elements must fit into the `uint16` type in order to be renderable.
+//
+// `effect_texture` must have positive dimensions. Its format must be either
+// `SRGB` or `SRGBA`. Its memory must be aligned for GL usage.
+mediapipe::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
+    const Environment& environment,                //
+    const absl::optional<Mesh3d>& effect_mesh_3d,  //
+    ImageFrame&& effect_texture);
+
+}  // namespace mediapipe::face_geometry
+
+#endif  // MEDIAPIPE_MODULES_FACE_GEOMETRY_LIBS_EFFECT_RENDERER_H_

mediapipe/modules/face_geometry/libs/geometry_pipeline.cc

@@ -0,0 +1,434 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/face_geometry/libs/geometry_pipeline.h"
+
+#include <cmath>
+#include <cstdint>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "Eigen/Core"
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/matrix_data.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
+#include "mediapipe/modules/face_geometry/libs/procrustes_solver.h"
+#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
+#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
+
+namespace mediapipe::face_geometry {
+namespace {
+
+struct PerspectiveCameraFrustum {
+  // NOTE: all arguments must be validated prior to calling this constructor.
+  PerspectiveCameraFrustum(const PerspectiveCamera& perspective_camera,
+                           int frame_width, int frame_height) {
+    static constexpr float kDegreesToRadians = 3.14159265358979323846f / 180.f;
+
+    const float height_at_near =
+        2.f * perspective_camera.near() *
+        std::tan(0.5f * kDegreesToRadians *
+                 perspective_camera.vertical_fov_degrees());
+
+    const float width_at_near = frame_width * height_at_near / frame_height;
+
+    left = -0.5f * width_at_near;
+    right = 0.5f * width_at_near;
+    bottom = -0.5f * height_at_near;
+    top = 0.5f * height_at_near;
+    near = perspective_camera.near();
+    far = perspective_camera.far();
+  }
+
+  float left;
+  float right;
+  float bottom;
+  float top;
+  float near;
+  float far;
+};
+
+class ScreenToMetricSpaceConverter {
+ public:
+  ScreenToMetricSpaceConverter(
+      OriginPointLocation origin_point_location,      //
+      Eigen::Matrix3Xf&& canonical_metric_landmarks,  //
+      Eigen::VectorXf&& landmark_weights,             //
+      std::unique_ptr<ProcrustesSolver> procrustes_solver)
+      : origin_point_location_(origin_point_location),
+        canonical_metric_landmarks_(std::move(canonical_metric_landmarks)),
+        landmark_weights_(std::move(landmark_weights)),
+        procrustes_solver_(std::move(procrustes_solver)) {}
+
+  // Converts `screen_landmark_list` into `metric_landmark_list` and estimates
+  // the `pose_transform_mat`.
+  //
+  // Here's the algorithm summary:
+  //
+  // (1) Project X- and Y- screen landmark coordinates at the Z near plane.
+  //
+  // (2) Estimate a canonical-to-runtime landmark set scale by running the
+  //     Procrustes solver using the screen runtime landmarks.
+  //
+  //     On this iteration, screen landmarks are used instead of unprojected
+  //     metric landmarks as it is not safe to unproject due to the relative
+  //     nature of the input screen landmark Z coordinate.
+  //
+  // (3) Use the canonical-to-runtime scale from (2) to unproject the screen
+  //     landmarks. The result is referenced as "intermediate landmarks" because
+  //     they are the first estimation of the resuling metric landmarks, but are
+  //     not quite there yet.
+  //
+  // (4) Estimate a canonical-to-runtime landmark set scale by running the
+  //     Procrustes solver using the intermediate runtime landmarks.
+  //
+  // (5) Use the product of the scale factors from (2) and (4) to unproject
+  //     the screen landmarks the second time. This is the second and the final
+  //     estimation of the metric landmarks.
+  //
+  // (6) Multiply each of the metric landmarks by the inverse pose
+  //     transformation matrix to align the runtime metric face landmarks with
+  //     the canonical metric face landmarks.
+  //
+  // Note: the input screen landmarks are in the left-handed coordinate system,
+  //       however any metric landmarks - including the canonical metric
+  //       landmarks, the final runtime metric landmarks and any intermediate
+  //       runtime metric landmarks - are in the right-handed coordinate system.
+  //
+  //       To keep the logic correct, the landmark set handedness is changed any
+  //       time the screen-to-metric semantic barrier is passed.
+  mediapipe::Status Convert(
+      const NormalizedLandmarkList& screen_landmark_list,  //
+      const PerspectiveCameraFrustum& pcf,                 //
+      LandmarkList& metric_landmark_list,                  //
+      Eigen::Matrix4f& pose_transform_mat) const {
+    RET_CHECK_EQ(screen_landmark_list.landmark_size(),
+                 canonical_metric_landmarks_.cols())
+        << "The number of landmarks doesn't match the number passed upon "
+           "initialization!";
+
+    Eigen::Matrix3Xf screen_landmarks;
+    ConvertLandmarkListToEigenMatrix(screen_landmark_list, screen_landmarks);
+
+    ProjectXY(pcf, screen_landmarks);
+    const float depth_offset = screen_landmarks.row(2).mean();
+
+    // 1st iteration: don't unproject XY because it's unsafe to do so due to
+    //                the relative nature of the Z coordinate. Instead, run the
+    //                first estimation on the projected XY and use that scale to
+    //                unproject for the 2nd iteration.
+    Eigen::Matrix3Xf intermediate_landmarks(screen_landmarks);
+    ChangeHandedness(intermediate_landmarks);
+
+    ASSIGN_OR_RETURN(const float first_iteration_scale,
+                     EstimateScale(intermediate_landmarks),
+                     _ << "Failed to estimate first iteration scale!");
+
+    // 2nd iteration: unproject XY using the scale from the 1st iteration.
+    intermediate_landmarks = screen_landmarks;
+    MoveAndRescaleZ(pcf, depth_offset, first_iteration_scale,
+                    intermediate_landmarks);
+    UnprojectXY(pcf, intermediate_landmarks);
+    ChangeHandedness(intermediate_landmarks);
+    ASSIGN_OR_RETURN(const float second_iteration_scale,
+                     EstimateScale(intermediate_landmarks),
+                     _ << "Failed to estimate second iteration scale!");
+
+    // Use the total scale to unproject the screen landmarks.
+    float total_scale = first_iteration_scale * second_iteration_scale;
+    MoveAndRescaleZ(pcf, depth_offset, total_scale, screen_landmarks);
+    UnprojectXY(pcf, screen_landmarks);
+    ChangeHandedness(screen_landmarks);
+
+    // At this point, screen landmarks are converted into metric landmarks.
+    Eigen::Matrix3Xf& metric_landmarks = screen_landmarks;
+
+    MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
+        canonical_metric_landmarks_, metric_landmarks, landmark_weights_,
+        pose_transform_mat))
+        << "Failed to estimate pose transform matrix!";
+
+    // Multiply each of the metric landmarks by the inverse pose transformation
+    // matrix to align the runtime metric face landmarks with the canonical
+    // metric face landmarks.
+    Eigen::Matrix4f inv_pose_transform_mat = pose_transform_mat.inverse();
+    auto inv_pose_rotation = inv_pose_transform_mat.leftCols(3).topRows(3);
+    auto inv_pose_translation = inv_pose_transform_mat.col(3).topRows(3);
+    metric_landmarks =
+        (inv_pose_rotation * metric_landmarks).colwise() + inv_pose_translation;
+
+    ConvertEigenMatrixToLandmarkList(metric_landmarks, metric_landmark_list);
+
+    return mediapipe::OkStatus();
+  }
+
+ private:
+  void ProjectXY(const PerspectiveCameraFrustum& pcf,
+                 Eigen::Matrix3Xf& landmarks) const {
+    float x_scale = pcf.right - pcf.left;
+    float y_scale = pcf.top - pcf.bottom;
+    float x_translation = pcf.left;
+    float y_translation = pcf.bottom;
+
+    if (origin_point_location_ == OriginPointLocation::TOP_LEFT_CORNER) {
+      landmarks.row(1) = 1.f - landmarks.row(1).array();
+    }
+
+    landmarks =
+        landmarks.array().colwise() * Eigen::Array3f(x_scale, y_scale, x_scale);
+    landmarks.colwise() += Eigen::Vector3f(x_translation, y_translation, 0.f);
+  }
+
+  mediapipe::StatusOr<float> EstimateScale(Eigen::Matrix3Xf& landmarks) const {
+    Eigen::Matrix4f transform_mat;
+    MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
+        canonical_metric_landmarks_, landmarks, landmark_weights_,
+        transform_mat))
+        << "Failed to estimate canonical-to-runtime landmark set transform!";
+
+    return transform_mat.col(0).norm();
+  }
+
+  static void MoveAndRescaleZ(const PerspectiveCameraFrustum& pcf,
+                              float depth_offset, float scale,
+                              Eigen::Matrix3Xf& landmarks) {
+    landmarks.row(2) =
+        (landmarks.array().row(2) - depth_offset + pcf.near) / scale;
+  }
+
+  static void UnprojectXY(const PerspectiveCameraFrustum& pcf,
+                          Eigen::Matrix3Xf& landmarks) {
+    landmarks.row(0) =
+        landmarks.row(0).cwiseProduct(landmarks.row(2)) / pcf.near;
+    landmarks.row(1) =
+        landmarks.row(1).cwiseProduct(landmarks.row(2)) / pcf.near;
+  }
+
+  static void ChangeHandedness(Eigen::Matrix3Xf& landmarks) {
+    landmarks.row(2) *= -1.f;
+  }
+
+  static void ConvertLandmarkListToEigenMatrix(
+      const NormalizedLandmarkList& landmark_list,
+      Eigen::Matrix3Xf& eigen_matrix) {
+    eigen_matrix = Eigen::Matrix3Xf(3, landmark_list.landmark_size());
+    for (int i = 0; i < landmark_list.landmark_size(); ++i) {
+      const auto& landmark = landmark_list.landmark(i);
+      eigen_matrix(0, i) = landmark.x();
+      eigen_matrix(1, i) = landmark.y();
+      eigen_matrix(2, i) = landmark.z();
+    }
+  }
+
+  static void ConvertEigenMatrixToLandmarkList(
+      const Eigen::Matrix3Xf& eigen_matrix, LandmarkList& landmark_list) {
+    landmark_list.Clear();
+
+    for (int i = 0; i < eigen_matrix.cols(); ++i) {
+      auto& landmark = *landmark_list.add_landmark();
+      landmark.set_x(eigen_matrix(0, i));
+      landmark.set_y(eigen_matrix(1, i));
+      landmark.set_z(eigen_matrix(2, i));
+    }
+  }
+
+  OriginPointLocation origin_point_location_;
+  Eigen::Matrix3Xf canonical_metric_landmarks_;
+  Eigen::VectorXf landmark_weights_;
+
+  std::unique_ptr<ProcrustesSolver> procrustes_solver_;
+};
+
+class GeometryPipelineImpl : public GeometryPipeline {
+ public:
+  GeometryPipelineImpl(
+      const PerspectiveCamera& perspective_camera,  //
+      const Mesh3d& canonical_mesh,                 //
+      uint32_t canonical_mesh_vertex_size,          //
+      uint32_t canonical_mesh_num_vertices,
+      uint32_t canonical_mesh_vertex_position_offset,
+      std::unique_ptr<ScreenToMetricSpaceConverter> space_converter)
+      : perspective_camera_(perspective_camera),
+        canonical_mesh_(canonical_mesh),
+        canonical_mesh_vertex_size_(canonical_mesh_vertex_size),
+        canonical_mesh_num_vertices_(canonical_mesh_num_vertices),
+        canonical_mesh_vertex_position_offset_(
+            canonical_mesh_vertex_position_offset),
+        space_converter_(std::move(space_converter)) {}
+
+  mediapipe::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
+      const std::vector<NormalizedLandmarkList>& multi_face_landmarks,
+      int frame_width, int frame_height) const override {
+    MP_RETURN_IF_ERROR(ValidateFrameDimensions(frame_width, frame_height))
+        << "Invalid frame dimensions!";
+
+    // Create a perspective camera frustum to be shared for geometry estimation
+    // per each face.
+    PerspectiveCameraFrustum pcf(perspective_camera_, frame_width,
+                                 frame_height);
+
+    std::vector<FaceGeometry> multi_face_geometry;
+
+    // From this point, the meaning of "face landmarks" is clarified further as
+    // "screen face landmarks". This is done do distinguish from "metric face
+    // landmarks" that are derived during the face geometry estimation process.
+    for (const NormalizedLandmarkList& screen_face_landmarks :
+         multi_face_landmarks) {
+      // Having a too compact screen landmark list will result in numerical
+      // instabilities, therefore such faces are filtered.
+      if (IsScreenLandmarkListTooCompact(screen_face_landmarks)) {
+        continue;
+      }
+
+      // Convert the screen landmarks into the metric landmarks and
+      // get the pose transformation matrix.
+      LandmarkList metric_face_landmarks;
+      Eigen::Matrix4f pose_transform_mat;
+      MP_RETURN_IF_ERROR(space_converter_->Convert(screen_face_landmarks, pcf,
+                                                   metric_face_landmarks,
+                                                   pose_transform_mat))
+          << "Failed to convert landmarks from the screen to the metric space!";
+
+      // Pack geometry data for this face.
+      FaceGeometry face_geometry;
+      Mesh3d* mutable_mesh = face_geometry.mutable_mesh();
+      // Copy the canonical face mesh as the face geometry mesh.
+      mutable_mesh->CopyFrom(canonical_mesh_);
+      // Replace XYZ vertex mesh coodinates with the metric landmark positions.
+      for (int i = 0; i < canonical_mesh_num_vertices_; ++i) {
+        uint32_t vertex_buffer_offset = canonical_mesh_vertex_size_ * i +
+                                        canonical_mesh_vertex_position_offset_;
+
+        mutable_mesh->set_vertex_buffer(vertex_buffer_offset,
+                                        metric_face_landmarks.landmark(i).x());
+        mutable_mesh->set_vertex_buffer(vertex_buffer_offset + 1,
+                                        metric_face_landmarks.landmark(i).y());
+        mutable_mesh->set_vertex_buffer(vertex_buffer_offset + 2,
+                                        metric_face_landmarks.landmark(i).z());
+      }
+      // Populate the face pose transformation matrix.
+      mediapipe::MatrixDataProtoFromMatrix(
+          pose_transform_mat, face_geometry.mutable_pose_transform_matrix());
+
+      multi_face_geometry.push_back(face_geometry);
+    }
+
+    return multi_face_geometry;
+  }
+
+ private:
+  static bool IsScreenLandmarkListTooCompact(
+      const NormalizedLandmarkList& screen_landmarks) {
+    float mean_x = 0.f;
+    float mean_y = 0.f;
+    for (int i = 0; i < screen_landmarks.landmark_size(); ++i) {
+      const auto& landmark = screen_landmarks.landmark(i);
+      mean_x += (landmark.x() - mean_x) / static_cast<float>(i + 1);
+      mean_y += (landmark.y() - mean_y) / static_cast<float>(i + 1);
+    }
+
+    float max_sq_dist = 0.f;
+    for (const auto& landmark : screen_landmarks.landmark()) {
+      const float d_x = landmark.x() - mean_x;
+      const float d_y = landmark.y() - mean_y;
+      max_sq_dist = std::max(max_sq_dist, d_x * d_x + d_y * d_y);
+    }
+
+    static constexpr float kIsScreenLandmarkListTooCompactThreshold = 1e-3f;
+    return std::sqrt(max_sq_dist) <= kIsScreenLandmarkListTooCompactThreshold;
+  }
+
+  const PerspectiveCamera perspective_camera_;
+  const Mesh3d canonical_mesh_;
+  const uint32_t canonical_mesh_vertex_size_;
+  const uint32_t canonical_mesh_num_vertices_;
+  const uint32_t canonical_mesh_vertex_position_offset_;
+
+  std::unique_ptr<ScreenToMetricSpaceConverter> space_converter_;
+};
+
+}  // namespace
+
+mediapipe::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
+    const Environment& environment, const GeometryPipelineMetadata& metadata) {
+  MP_RETURN_IF_ERROR(ValidateEnvironment(environment))
+      << "Invalid environment!";
+  MP_RETURN_IF_ERROR(ValidateGeometryPipelineMetadata(metadata))
+      << "Invalid geometry pipeline metadata!";
+
+  const auto& canonical_mesh = metadata.canonical_mesh();
+  RET_CHECK(HasVertexComponent(canonical_mesh.vertex_type(),
+                               VertexComponent::POSITION))
+      << "Canonical face mesh must have the `POSITION` vertex component!";
+  RET_CHECK(HasVertexComponent(canonical_mesh.vertex_type(),
+                               VertexComponent::TEX_COORD))
+      << "Canonical face mesh must have the `TEX_COORD` vertex component!";
+
+  uint32_t canonical_mesh_vertex_size =
+      GetVertexSize(canonical_mesh.vertex_type());
+  uint32_t canonical_mesh_num_vertices =
+      canonical_mesh.vertex_buffer_size() / canonical_mesh_vertex_size;
+  uint32_t canonical_mesh_vertex_position_offset =
+      GetVertexComponentOffset(canonical_mesh.vertex_type(),
+                               VertexComponent::POSITION)
+          .ValueOrDie();
+
+  // Put the Procrustes landmark basis into Eigen matrices for an easier access.
+  Eigen::Matrix3Xf canonical_metric_landmarks =
+      Eigen::Matrix3Xf::Zero(3, canonical_mesh_num_vertices);
+  Eigen::VectorXf landmark_weights =
+      Eigen::VectorXf::Zero(canonical_mesh_num_vertices);
+
+  for (int i = 0; i < canonical_mesh_num_vertices; ++i) {
+    uint32_t vertex_buffer_offset =
+        canonical_mesh_vertex_size * i + canonical_mesh_vertex_position_offset;
+
+    canonical_metric_landmarks(0, i) =
+        canonical_mesh.vertex_buffer(vertex_buffer_offset);
+    canonical_metric_landmarks(1, i) =
+        canonical_mesh.vertex_buffer(vertex_buffer_offset + 1);
+    canonical_metric_landmarks(2, i) =
+        canonical_mesh.vertex_buffer(vertex_buffer_offset + 2);
+  }
+
+  for (const WeightedLandmarkRef& wlr : metadata.procrustes_landmark_basis()) {
+    uint32_t landmark_id = wlr.landmark_id();
+    landmark_weights(landmark_id) = wlr.weight();
+  }
+
+  std::unique_ptr<GeometryPipeline> result =
+      absl::make_unique<GeometryPipelineImpl>(
+          environment.perspective_camera(), canonical_mesh,
+          canonical_mesh_vertex_size, canonical_mesh_num_vertices,
+          canonical_mesh_vertex_position_offset,
+          absl::make_unique<ScreenToMetricSpaceConverter>(
+              environment.origin_point_location(),
+              std::move(canonical_metric_landmarks),
+              std::move(landmark_weights),
+              CreateFloatPrecisionProcrustesSolver()));
+
+  return result;
+}
+
+}  // namespace mediapipe::face_geometry

mediapipe/modules/face_geometry/libs/geometry_pipeline.h

@@ -0,0 +1,67 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_GEOMETRY_PIPELINE_H_
+#define MEDIAPIPE_FACE_GEOMETRY_LIBS_GEOMETRY_PIPELINE_H_
+
+#include <memory>
+#include <vector>
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
+
+namespace mediapipe::face_geometry {
+
+// Encapsulates a stateless estimator of facial geometry in a Metric space based
+// on the normalized face landmarks in the Screen space.
+class GeometryPipeline {
+ public:
+  virtual ~GeometryPipeline() = default;
+
+  // Estimates geometry data for multiple faces.
+  //
+  // Returns an error status if any of the passed arguments is invalid.
+  //
+  // The result includes face geometry data for a subset of the input faces,
+  // however geometry data for some faces might be missing. This may happen if
+  // it'd be unstable to estimate the facial geometry based on a corresponding
+  // face landmark list for any reason (for example, if the landmark list is too
+  // compact).
+  //
+  // Each face landmark list must have the same number of landmarks as was
+  // passed upon initialization via the canonical face mesh (as a part of the
+  // geometry pipeline metadata).
+  //
+  // Both `frame_width` and `frame_height` must be positive.
+  virtual mediapipe::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
+      const std::vector<NormalizedLandmarkList>& multi_face_landmarks,
+      int frame_width, int frame_height) const = 0;
+};
+
+// Creates an instance of `GeometryPipeline`.
+//
+// Both `environment` and `metadata` must be valid (for details, please refer to
+// the proto message definition comments and/or `validation_utils.h/cc`).
+//
+// Canonical face mesh (defined as a part of `metadata`) must have the
+// `POSITION` and the `TEX_COORD` vertex components.
+mediapipe::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
+    const Environment& environment, const GeometryPipelineMetadata& metadata);
+
+}  // namespace mediapipe::face_geometry
+
+#endif  // MEDIAPIPE_FACE_GEOMETRY_LIBS_GEOMETRY_PIPELINE_H_

mediapipe/modules/face_geometry/libs/mesh_3d_utils.cc

@@ -0,0 +1,103 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
+
+#include <cstdint>
+#include <cstdlib>
+
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
+
+namespace mediapipe::face_geometry {
+namespace {
+
+bool HasVertexComponentVertexPT(VertexComponent vertex_component) {
+  switch (vertex_component) {
+    case VertexComponent::POSITION:
+    case VertexComponent::TEX_COORD:
+      return true;
+
+    default:
+      return false;
+  }
+}
+
+uint32_t GetVertexComponentSizeVertexPT(VertexComponent vertex_component) {
+  switch (vertex_component) {
+    case VertexComponent::POSITION:
+      return 3;
+    case VertexComponent::TEX_COORD:
+      return 2;
+  }
+}
+
+uint32_t GetVertexComponentOffsetVertexPT(VertexComponent vertex_component) {
+  switch (vertex_component) {
+    case VertexComponent::POSITION:
+      return 0;
+    case VertexComponent::TEX_COORD:
+      return GetVertexComponentSizeVertexPT(VertexComponent::POSITION);
+  }
+}
+
+}  // namespace
+
+std::size_t GetVertexSize(Mesh3d::VertexType vertex_type) {
+  switch (vertex_type) {
+    case Mesh3d::VERTEX_PT:
+      return GetVertexComponentSizeVertexPT(VertexComponent::POSITION) +
+             GetVertexComponentSizeVertexPT(VertexComponent::TEX_COORD);
+  }
+}
+
+std::size_t GetPrimitiveSize(Mesh3d::PrimitiveType primitive_type) {
+  switch (primitive_type) {
+    case Mesh3d::TRIANGLE:
+      return 3;
+  }
+}
+
+bool HasVertexComponent(Mesh3d::VertexType vertex_type,
+                        VertexComponent vertex_component) {
+  switch (vertex_type) {
+    case Mesh3d::VERTEX_PT:
+      return HasVertexComponentVertexPT(vertex_component);
+  }
+}
+
+mediapipe::StatusOr<uint32_t> GetVertexComponentOffset(
+    Mesh3d::VertexType vertex_type, VertexComponent vertex_component) {
+  RET_CHECK(HasVertexComponentVertexPT(vertex_component))
+      << "A given vertex type doesn't have the requested component!";
+
+  switch (vertex_type) {
+    case Mesh3d::VERTEX_PT:
+      return GetVertexComponentOffsetVertexPT(vertex_component);
+  }
+}
+
+mediapipe::StatusOr<uint32_t> GetVertexComponentSize(
+    Mesh3d::VertexType vertex_type, VertexComponent vertex_component) {
+  RET_CHECK(HasVertexComponentVertexPT(vertex_component))
+      << "A given vertex type doesn't have the requested component!";
+
+  switch (vertex_type) {
+    case Mesh3d::VERTEX_PT:
+      return GetVertexComponentSizeVertexPT(vertex_component);
+  }
+}
+
+}  // namespace mediapipe::face_geometry

mediapipe/modules/face_geometry/libs/mesh_3d_utils.h

@@ -0,0 +1,51 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_MESH_3D_UTILS_H_
+#define MEDIAPIPE_FACE_GEOMETRY_LIBS_MESH_3D_UTILS_H_
+
+#include <cstdint>
+#include <cstdlib>
+
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
+
+namespace mediapipe::face_geometry {
+
+enum class VertexComponent { POSITION, TEX_COORD };
+
+std::size_t GetVertexSize(Mesh3d::VertexType vertex_type);
+
+std::size_t GetPrimitiveSize(Mesh3d::PrimitiveType primitive_type);
+
+bool HasVertexComponent(Mesh3d::VertexType vertex_type,
+                        VertexComponent vertex_component);
+
+// Computes the vertex component offset.
+//
+// Returns an error status if a given vertex type doesn't have the requested
+// component.
+mediapipe::StatusOr<uint32_t> GetVertexComponentOffset(
+    Mesh3d::VertexType vertex_type, VertexComponent vertex_component);
+
+// Computes the vertex component size.
+//
+// Returns an error status if a given vertex type doesn't have the requested
+// component.
+mediapipe::StatusOr<uint32_t> GetVertexComponentSize(
+    Mesh3d::VertexType vertex_type, VertexComponent vertex_component);
+
+}  // namespace mediapipe::face_geometry
+
+#endif  // MEDIAPIPE_FACE_GEOMETRY_LIBS_MESH_3D_UTILS_H_

mediapipe/modules/face_geometry/libs/procrustes_solver.cc

@@ -0,0 +1,266 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/face_geometry/libs/procrustes_solver.h"
+
+#include <cmath>
+#include <memory>
+
+#include "Eigen/Dense"
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/port/statusor.h"
+
+namespace mediapipe {
+namespace face_geometry {
+namespace {
+
+class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
+ public:
+  FloatPrecisionProcrustesSolver() = default;
+
+  mediapipe::Status SolveWeightedOrthogonalProblem(
+      const Eigen::Matrix3Xf& source_points,  //
+      const Eigen::Matrix3Xf& target_points,  //
+      const Eigen::VectorXf& point_weights,
+      Eigen::Matrix4f& transform_mat) const override {
+    // Validate inputs.
+    MP_RETURN_IF_ERROR(ValidateInputPoints(source_points, target_points))
+        << "Failed to validate weighted orthogonal problem input points!";
+    MP_RETURN_IF_ERROR(
+        ValidatePointWeights(source_points.cols(), point_weights))
+        << "Failed to validate weighted orthogonal problem point weights!";
+
+    // Extract square root from the point weights.
+    Eigen::VectorXf sqrt_weights = ExtractSquareRoot(point_weights);
+
+    // Try to solve the WEOP problem.
+    MP_RETURN_IF_ERROR(InternalSolveWeightedOrthogonalProblem(
+        source_points, target_points, sqrt_weights, transform_mat))
+        << "Failed to solve the WEOP problem!";
+
+    return mediapipe::OkStatus();
+  }
+
+ private:
+  static constexpr float kAbsoluteErrorEps = 1e-9f;
+
+  static mediapipe::Status ValidateInputPoints(
+      const Eigen::Matrix3Xf& source_points,
+      const Eigen::Matrix3Xf& target_points) {
+    RET_CHECK_GT(source_points.cols(), 0)
+        << "The number of source points must be positive!";
+
+    RET_CHECK_EQ(source_points.cols(), target_points.cols())
+        << "The number of source and target points must be equal!";
+
+    return mediapipe::OkStatus();
+  }
+
+  static mediapipe::Status ValidatePointWeights(
+      int num_points, const Eigen::VectorXf& point_weights) {
+    RET_CHECK_GT(point_weights.size(), 0)
+        << "The number of point weights must be positive!";
+
+    RET_CHECK_EQ(point_weights.size(), num_points)
+        << "The number of points and point weights must be equal!";
+
+    float total_weight = 0.f;
+    for (int i = 0; i < num_points; ++i) {
+      RET_CHECK_GE(point_weights(i), 0.f)
+          << "Each point weight must be non-negative!";
+
+      total_weight += point_weights(i);
+    }
+
+    RET_CHECK_GT(total_weight, kAbsoluteErrorEps)
+        << "The total point weight is too small!";
+
+    return mediapipe::OkStatus();
+  }
+
+  static Eigen::VectorXf ExtractSquareRoot(
+      const Eigen::VectorXf& point_weights) {
+    Eigen::VectorXf sqrt_weights(point_weights);
+    for (int i = 0; i < sqrt_weights.size(); ++i) {
+      sqrt_weights(i) = std::sqrt(sqrt_weights(i));
+    }
+
+    return sqrt_weights;
+  }
+
+  // Combines a 3x3 rotation-and-scale matrix and a 3x1 translation vector into
+  // a single 4x4 transformation matrix.
+  static Eigen::Matrix4f CombineTransformMatrix(const Eigen::Matrix3f& r_and_s,
+                                                const Eigen::Vector3f& t) {
+    Eigen::Matrix4f result = Eigen::Matrix4f::Identity();
+    result.leftCols(3).topRows(3) = r_and_s;
+    result.col(3).topRows(3) = t;
+
+    return result;
+  }
+
+  // The weighted problem is thoroughly addressed in Section 2.4 of:
+  // D. Akca, Generalized Procrustes analysis and its applications
+  // in photogrammetry, 2003, https://doi.org/10.3929/ethz-a-004656648
+  //
+  // Notable differences in the code presented here are:
+  //
+  //   * In the paper, the weights matrix W_p is Cholesky-decomposed as Q^T Q.
+  //     Our W_p is diagonal (equal to diag(sqrt_weights^2)),
+  //     so we can just set Q = diag(sqrt_weights) instead.
+  //
+  //   * In the paper, the problem is presented as
+  //     (for W_k = I and W_p = tranposed(Q) Q):
+  //     || Q (c A T + j tranposed(t) - B) || -> min.
+  //
+  //     We reformulate it as an equivalent minimization of the transpose's
+  //     norm:
+  //     || (c tranposed(T) tranposed(A) - tranposed(B)) tranposed(Q) || -> min,
+  //     where tranposed(A) and tranposed(B) are the source and the target point
+  //     clouds, respectively, c tranposed(T) is the rotation+scaling R sought
+  //     for, and Q is diag(sqrt_weights).
+  //
+  //     Most of the derivations are therefore transposed.
+  //
+  // Note: the output `transform_mat` argument is used instead of `StatusOr<>`
+  // return type in order to avoid Eigen memory alignment issues. Details:
+  // https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
+  static mediapipe::Status InternalSolveWeightedOrthogonalProblem(
+      const Eigen::Matrix3Xf& sources, const Eigen::Matrix3Xf& targets,
+      const Eigen::VectorXf& sqrt_weights, Eigen::Matrix4f& transform_mat) {
+    // tranposed(A_w).
+    Eigen::Matrix3Xf weighted_sources =
+        sources.array().rowwise() * sqrt_weights.array().transpose();
+    // tranposed(B_w).
+    Eigen::Matrix3Xf weighted_targets =
+        targets.array().rowwise() * sqrt_weights.array().transpose();
+
+    // w = tranposed(j_w) j_w.
+    float total_weight = sqrt_weights.cwiseProduct(sqrt_weights).sum();
+
+    // Let C = (j_w tranposed(j_w)) / (tranposed(j_w) j_w).
+    // Note that C = tranposed(C), hence (I - C) = tranposed(I - C).
+    //
+    // tranposed(A_w) C = tranposed(A_w) j_w tranposed(j_w) / w =
+    // (tranposed(A_w) j_w) tranposed(j_w) / w = c_w tranposed(j_w),
+    //
+    // where c_w = tranposed(A_w) j_w / w is a k x 1 vector calculated here:
+    Eigen::Matrix3Xf twice_weighted_sources =
+        weighted_sources.array().rowwise() * sqrt_weights.array().transpose();
+    Eigen::Vector3f source_center_of_mass =
+        twice_weighted_sources.rowwise().sum() / total_weight;
+    // tranposed((I - C) A_w) = tranposed(A_w) (I - C) =
+    // tranposed(A_w) - tranposed(A_w) C = tranposed(A_w) - c_w tranposed(j_w).
+    Eigen::Matrix3Xf centered_weighted_sources =
+        weighted_sources - source_center_of_mass * sqrt_weights.transpose();
+
+    Eigen::Matrix3f rotation;
+    MP_RETURN_IF_ERROR(ComputeOptimalRotation(
+        weighted_targets * centered_weighted_sources.transpose(), rotation))
+        << "Failed to compute the optimal rotation!";
+    ASSIGN_OR_RETURN(
+        float scale,
+        ComputeOptimalScale(centered_weighted_sources, weighted_sources,
+                            weighted_targets, rotation),
+        _ << "Failed to compute the optimal scale!");
+
+    // R = c tranposed(T).
+    Eigen::Matrix3f rotation_and_scale = scale * rotation;
+
+    // Compute optimal translation for the weighted problem.
+
+    // tranposed(B_w - c A_w T) = tranposed(B_w) - R tranposed(A_w) in (54).
+    const auto pointwise_diffs =
+        weighted_targets - rotation_and_scale * weighted_sources;
+    // Multiplication by j_w is a respectively weighted column sum.
+    // (54) from the paper.
+    const auto weighted_pointwise_diffs =
+        pointwise_diffs.array().rowwise() * sqrt_weights.array().transpose();
+    Eigen::Vector3f translation =
+        weighted_pointwise_diffs.rowwise().sum() / total_weight;
+
+    transform_mat = CombineTransformMatrix(rotation_and_scale, translation);
+
+    return mediapipe::OkStatus();
+  }
+
+  // `design_matrix` is a transposed LHS of (51) in the paper.
+  //
+  // Note: the output `rotation` argument is used instead of `StatusOr<>`
+  // return type in order to avoid Eigen memory alignment issues. Details:
+  // https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
+  static mediapipe::Status ComputeOptimalRotation(
+      const Eigen::Matrix3f& design_matrix, Eigen::Matrix3f& rotation) {
+    RET_CHECK_GT(design_matrix.norm(), kAbsoluteErrorEps)
+        << "Design matrix norm is too small!";
+
+    Eigen::JacobiSVD<Eigen::Matrix3f> svd(
+        design_matrix, Eigen::ComputeFullU | Eigen::ComputeFullV);
+
+    Eigen::Matrix3f postrotation = svd.matrixU();
+    Eigen::Matrix3f prerotation = svd.matrixV().transpose();
+
+    // Disallow reflection by ensuring that det(`rotation`) = +1 (and not -1),
+    // see "4.6 Constrained orthogonal Procrustes problems"
+    // in the Gower & Dijksterhuis's book "Procrustes Analysis".
+    // We flip the sign of the least singular value along with a column in W.
+    //
+    // Note that now the sum of singular values doesn't work for scale
+    // estimation due to this sign flip.
+    if (postrotation.determinant() * prerotation.determinant() <
+        static_cast<float>(0)) {
+      postrotation.col(2) *= static_cast<float>(-1);
+    }
+
+    // Transposed (52) from the paper.
+    rotation = postrotation * prerotation;
+    return mediapipe::OkStatus();
+  }
+
+  static mediapipe::StatusOr<float> ComputeOptimalScale(
+      const Eigen::Matrix3Xf& centered_weighted_sources,
+      const Eigen::Matrix3Xf& weighted_sources,
+      const Eigen::Matrix3Xf& weighted_targets,
+      const Eigen::Matrix3f& rotation) {
+    // tranposed(T) tranposed(A_w) (I - C).
+    const auto rotated_centered_weighted_sources =
+        rotation * centered_weighted_sources;
+    // Use the identity trace(A B) = sum(A * B^T)
+    // to avoid building large intermediate matrices (* is Hadamard product).
+    // (53) from the paper.
+    float numerator =
+        rotated_centered_weighted_sources.cwiseProduct(weighted_targets).sum();
+    float denominator =
+        centered_weighted_sources.cwiseProduct(weighted_sources).sum();
+
+    RET_CHECK_GT(denominator, kAbsoluteErrorEps)
+        << "Scale expression denominator is too small!";
+    RET_CHECK_GT(numerator / denominator, kAbsoluteErrorEps)
+        << "Scale is too small!";
+
+    return numerator / denominator;
+  }
+};
+
+}  // namespace
+
+std::unique_ptr<ProcrustesSolver> CreateFloatPrecisionProcrustesSolver() {
+  return absl::make_unique<FloatPrecisionProcrustesSolver>();
+}
+
+}  // namespace face_geometry
+}  // namespace mediapipe

mediapipe/modules/face_geometry/libs/procrustes_solver.h

@@ -0,0 +1,70 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_PROCRUSTES_SOLVER_H_
+#define MEDIAPIPE_FACE_GEOMETRY_LIBS_PROCRUSTES_SOLVER_H_
+
+#include <memory>
+
+#include "Eigen/Dense"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe::face_geometry {
+
+// Encapsulates a stateless solver for the Weighted Extended Orthogonal
+// Procrustes (WEOP) Problem, as defined in Section 2.4 of
+// https://doi.org/10.3929/ethz-a-004656648.
+//
+// Given the source and the target point clouds, the algorithm estimates
+// a 4x4 transformation matrix featuring the following semantic components:
+//
+//   * Uniform scale
+//   * Rotation
+//   * Translation
+//
+// The matrix maps the source point cloud into the target point cloud minimizing
+// the Mean Squared Error.
+class ProcrustesSolver {
+ public:
+  virtual ~ProcrustesSolver() = default;
+
+  // Solves the Weighted Extended Orthogonal Procrustes (WEOP) Problem.
+  //
+  // All `source_points`, `target_points` and `point_weights` must define the
+  // same number of points. Elements of `point_weights` must be non-negative.
+  //
+  // A too small diameter of either of the point clouds will likely lead to
+  // numerical instabilities and failure to estimate the transformation.
+  //
+  // A too small point cloud total weight will likely lead to numerical
+  // instabilities and failure to estimate the transformation too.
+  //
+  // Small point coordinate deviation for either of the point cloud will likely
+  // result in a failure as it will make the solution very unstable if possible.
+  //
+  // Note: the output `transform_mat` argument is used instead of `StatusOr<>`
+  // return type in order to avoid Eigen memory alignment issues. Details:
+  // https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
+  virtual mediapipe::Status SolveWeightedOrthogonalProblem(
+      const Eigen::Matrix3Xf& source_points,  //
+      const Eigen::Matrix3Xf& target_points,  //
+      const Eigen::VectorXf& point_weights,   //
+      Eigen::Matrix4f& transform_mat) const = 0;
+};
+
+std::unique_ptr<ProcrustesSolver> CreateFloatPrecisionProcrustesSolver();
+
+}  // namespace mediapipe::face_geometry
+
+#endif  // MEDIAPIPE_FACE_GEOMETRY_LIBS_PROCRUSTES_SOLVER_H_

mediapipe/modules/face_geometry/libs/validation_utils.cc

@@ -0,0 +1,126 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
+
+#include <cstdint>
+#include <cstdlib>
+
+#include "mediapipe/framework/formats/matrix_data.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
+#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
+
+namespace mediapipe::face_geometry {
+
+mediapipe::Status ValidatePerspectiveCamera(
+    const PerspectiveCamera& perspective_camera) {
+  static constexpr float kAbsoluteErrorEps = 1e-9f;
+
+  RET_CHECK_GT(perspective_camera.near(), kAbsoluteErrorEps)
+      << "Near Z must be greater than 0 with a margin of 10^{-9}!";
+
+  RET_CHECK_GT(perspective_camera.far(),
+               perspective_camera.near() + kAbsoluteErrorEps)
+      << "Far Z must be greater than Near Z with a margin of 10^{-9}!";
+
+  RET_CHECK_GT(perspective_camera.vertical_fov_degrees(), kAbsoluteErrorEps)
+      << "Vertical FOV must be positive with a margin of 10^{-9}!";
+
+  RET_CHECK_LT(perspective_camera.vertical_fov_degrees() + kAbsoluteErrorEps,
+               180.f)
+      << "Vertical FOV must be less than 180 degrees with a margin of 10^{-9}";
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status ValidateEnvironment(const Environment& environment) {
+  MP_RETURN_IF_ERROR(
+      ValidatePerspectiveCamera(environment.perspective_camera()))
+      << "Invalid perspective camera!";
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status ValidateMesh3d(const Mesh3d& mesh_3d) {
+  const std::size_t vertex_size = GetVertexSize(mesh_3d.vertex_type());
+  const std::size_t primitive_type = GetPrimitiveSize(mesh_3d.primitive_type());
+
+  RET_CHECK_EQ(mesh_3d.vertex_buffer_size() % vertex_size, 0)
+      << "Vertex buffer size must a multiple of the vertex size!";
+
+  RET_CHECK_EQ(mesh_3d.index_buffer_size() % primitive_type, 0)
+      << "Index buffer size must a multiple of the primitive size!";
+
+  const int num_vertices = mesh_3d.vertex_buffer_size() / vertex_size;
+  for (uint32_t idx : mesh_3d.index_buffer()) {
+    RET_CHECK_LT(idx, num_vertices)
+        << "All mesh indices must refer to an existing vertex!";
+  }
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status ValidateFaceGeometry(const FaceGeometry& face_geometry) {
+  MP_RETURN_IF_ERROR(ValidateMesh3d(face_geometry.mesh())) << "Invalid mesh!";
+
+  static constexpr char kInvalid4x4MatrixMessage[] =
+      "Pose transformation matrix must be a 4x4 matrix!";
+
+  const MatrixData& pose_transform_matrix =
+      face_geometry.pose_transform_matrix();
+  RET_CHECK_EQ(pose_transform_matrix.rows(), 4) << kInvalid4x4MatrixMessage;
+  RET_CHECK_EQ(pose_transform_matrix.rows(), 4) << kInvalid4x4MatrixMessage;
+  RET_CHECK_EQ(pose_transform_matrix.packed_data_size(), 16)
+      << kInvalid4x4MatrixMessage;
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status ValidateGeometryPipelineMetadata(
+    const GeometryPipelineMetadata& metadata) {
+  MP_RETURN_IF_ERROR(ValidateMesh3d(metadata.canonical_mesh()))
+      << "Invalid canonical mesh!";
+
+  RET_CHECK_GT(metadata.procrustes_landmark_basis_size(), 0)
+
+      << "Procrustes landmark basis must be non-empty!";
+
+  const int num_vertices =
+      metadata.canonical_mesh().vertex_buffer_size() /
+      GetVertexSize(metadata.canonical_mesh().vertex_type());
+  for (const WeightedLandmarkRef& wlr : metadata.procrustes_landmark_basis()) {
+    RET_CHECK_LT(wlr.landmark_id(), num_vertices)
+        << "All Procrustes basis indices must refer to an existing canonical "
+           "mesh vertex!";
+
+    RET_CHECK_GE(wlr.weight(), 0.f)
+        << "All Procrustes basis landmarks must have a non-negative weight!";
+  }
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status ValidateFrameDimensions(int frame_width, int frame_height) {
+  RET_CHECK_GT(frame_width, 0) << "Frame width must be positive!";
+  RET_CHECK_GT(frame_height, 0) << "Frame height must be positive!";
+
+  return mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe::face_geometry

mediapipe/modules/face_geometry/libs/validation_utils.h

@@ -0,0 +1,70 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_VALIDATION_UTILS_H_
+#define MEDIAPIPE_FACE_GEOMETRY_LIBS_VALIDATION_UTILS_H_
+
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
+#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
+
+namespace mediapipe::face_geometry {
+
+// Validates `perspective_camera`.
+//
+// Near Z must be greater than 0 with a margin of `1e-9`.
+// Far Z must be greater than Near Z with a margin of `1e-9`.
+// Vertical FOV must be in range (0, 180) with a margin of `1e-9` on the range
+// edges.
+mediapipe::Status ValidatePerspectiveCamera(
+    const PerspectiveCamera& perspective_camera);
+
+// Validates `environment`.
+//
+// Environment's perspective camera must be valid.
+mediapipe::Status ValidateEnvironment(const Environment& environment);
+
+// Validates `mesh_3d`.
+//
+// Mesh vertex buffer size must a multiple of the vertex size.
+// Mesh index buffer size must a multiple of the primitive size.
+// All mesh indices must reference an existing mesh vertex.
+mediapipe::Status ValidateMesh3d(const Mesh3d& mesh_3d);
+
+// Validates `face_geometry`.
+//
+// Face mesh must be valid.
+// Face pose transformation matrix must be a 4x4 matrix.
+mediapipe::Status ValidateFaceGeometry(const FaceGeometry& face_geometry);
+
+// Validates `metadata`.
+//
+// Canonical face mesh must be valid.
+// Procrustes landmark basis must be non-empty.
+// All Procrustes basis indices must reference an existing canonical mesh
+// vertex.
+// All Procrustes basis landmarks must have a non-negative weight.
+mediapipe::Status ValidateGeometryPipelineMetadata(
+    const GeometryPipelineMetadata& metadata);
+
+// Validates frame dimensions.
+//
+// Both frame width and frame height must be positive.
+mediapipe::Status ValidateFrameDimensions(int frame_width, int frame_height);
+
+}  // namespace mediapipe::face_geometry
+
+#endif  // MEDIAPIPE_FACE_GEOMETRY_LIBS_VALIDATION_UTILS_H_

mediapipe/modules/face_geometry/protos/BUILD

@@ -0,0 +1,46 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+mediapipe_proto_library(
+    name = "environment_proto",
+    srcs = ["environment.proto"],
+)
+
+mediapipe_proto_library(
+    name = "face_geometry_proto",
+    srcs = ["face_geometry.proto"],
+    deps = [
+        ":mesh_3d_proto",
+        "//mediapipe/framework/formats:matrix_data_proto",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "geometry_pipeline_metadata_proto",
+    srcs = ["geometry_pipeline_metadata.proto"],
+    deps = [
+        ":mesh_3d_proto",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "mesh_3d_proto",
+    srcs = ["mesh_3d.proto"],
+)

mediapipe/modules/face_geometry/protos/environment.proto

@@ -0,0 +1,84 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.face_geometry;
+
+option java_package = "com.google.mediapipe.modules.facegeometry";
+option java_outer_classname = "EnvironmentProto";
+
+// Defines the (0, 0) origin point location of the environment.
+//
+// The variation in the origin point location can be traced back to the memory
+// layout of the camera video frame buffers.
+//
+// Usually, the memory layout for most CPU (and also some GPU) camera video
+// frame buffers results in having the (0, 0) origin point located in the
+// Top Left corner.
+//
+// On the contrary, the memory layout for most GPU camera video frame buffers
+// results in having the (0, 0) origin point located in the Bottom Left corner.
+//
+// Let's consider the following example:
+//
+// (A) ---------------+
+//               ___  |
+//  |     (1)    | |  |
+//  |     / \    | |  |
+//  |    |---|===|-|  |
+//  |    |---|   | |  |
+//  |   /     \  | |  |
+//  |  |       | | |  |
+//  |  |  (2)  |=| |  |
+//  |  |       | | |  |
+//  |  |_______| |_|  |
+//  |   |@| |@|  | |  |
+//  | ___________|_|_ |
+//                    |
+// (B) ---------------+
+//
+// On this example, (1) and (2) have the same X coordinate regardless of the
+// origin point location. However, having the origin point located at (A)
+// (Top Left corner) results in (1) having a smaller Y coordinate if compared to
+// (2). Similarly, having the origin point located at (B) (Bottom Left corner)
+// results in (1) having a greater Y coordinate if compared to (2).
+//
+// Providing the correct origin point location for your environment and making
+// sure all the input landmarks are in-sync with this location is crucial
+// for receiving the correct output face geometry and visual renders.
+enum OriginPointLocation {
+  BOTTOM_LEFT_CORNER = 1;
+  TOP_LEFT_CORNER = 2;
+}
+
+// The perspective camera is defined through its vertical FOV angle and the
+// Z-clipping planes. The aspect ratio is a runtime variable for the face
+// geometry module and should be provided alongside the face landmarks in order
+// to estimate the face geometry on a given frame.
+//
+// More info on Perspective Cameras:
+// http://www.songho.ca/opengl/gl_projectionmatrix.html#perspective
+message PerspectiveCamera {
+  // `0 < vertical_fov_degrees < 180`.
+  optional float vertical_fov_degrees = 1;
+  // `0 < near < far`.
+  optional float near = 2;
+  optional float far = 3;
+}
+
+message Environment {
+  optional OriginPointLocation origin_point_location = 1;
+  optional PerspectiveCamera perspective_camera = 2;
+}

mediapipe/modules/face_geometry/protos/face_geometry.proto

@@ -0,0 +1,51 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.face_geometry;
+
+import "mediapipe/framework/formats/matrix_data.proto";
+import "mediapipe/modules/face_geometry/protos/mesh_3d.proto";
+
+option java_package = "com.google.mediapipe.modules.facegeometry";
+option java_outer_classname = "FaceGeometryProto";
+
+// Defines the face geometry pipeline estimation result format.
+message FaceGeometry {
+  // Defines a mesh surface for a face. The face mesh vertex IDs are the same as
+  // the face landmark IDs.
+  //
+  // XYZ coordinates exist in the right-handed Metric 3D space configured by an
+  // environment. UV coodinates are taken from the canonical face mesh model.
+  //
+  // XY coordinates are guaranteed to match the screen positions of
+  // the input face landmarks after (1) being multiplied by the face pose
+  // transformation matrix and then (2) being projected with a perspective
+  // camera matrix of the same environment.
+  optional Mesh3d mesh = 1;
+
+  // Defines a face pose transformation matrix, which provides mapping from
+  // the static canonical face model to the runtime face. Tries to distinguish
+  // a head pose change from a facial expression change and to only reflect the
+  // former.
+  //
+  // Is a 4x4 matrix and contains only the following components:
+  //   * Uniform scale
+  //   * Rotation
+  //   * Translation
+  //
+  // The last row is guaranteed to be `[0 0 0 1]`.
+  optional MatrixData pose_transform_matrix = 2;
+}

mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.proto

@@ -0,0 +1,46 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.face_geometry;
+
+import "mediapipe/modules/face_geometry/protos/mesh_3d.proto";
+
+option java_package = "com.google.mediapipe.modules.facegeometry";
+option java_outer_classname = "GeometryPipelineMetadataProto";
+
+message WeightedLandmarkRef {
+  // Defines the landmark ID. References an existing face landmark ID.
+  optional uint32 landmark_id = 1;
+  // Defines the landmark weight. The larger the weight the more influence this
+  // landmark has in the basis.
+  //
+  // Is positive.
+  optional float weight = 2;
+}
+
+message GeometryPipelineMetadata {
+  // Defines a mesh surface for a canonical face. The canonical face mesh vertex
+  // IDs are the same as the face landmark IDs.
+  //
+  // XYZ coordinates are defined in centimeter units.
+  optional Mesh3d canonical_mesh = 1;
+  // Defines a weighted landmark basis for running the Procrustes solver
+  // algorithm inside the geometry pipeline.
+  //
+  // A good basis sets face landmark weights in way to distinguish a head pose
+  // change from a facial expression change and to only respond to the former.
+  repeated WeightedLandmarkRef procrustes_landmark_basis = 2;
+}

mediapipe/modules/face_geometry/protos/mesh_3d.proto

@@ -0,0 +1,41 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.face_geometry;
+
+option java_package = "com.google.mediapipe.modules.facegeometry";
+option java_outer_classname = "Mesh3dProto";
+
+message Mesh3d {
+  enum VertexType {
+    // Is defined by 5 coordinates: Position (XYZ) + Texture coordinate (UV).
+    VERTEX_PT = 0;
+  }
+
+  enum PrimitiveType {
+    // Is defined by 3 indices: triangle vertex IDs.
+    TRIANGLE = 0;
+  }
+
+  optional VertexType vertex_type = 1;
+  optional PrimitiveType primitive_type = 2;
+  // Vertex buffer size is a multiple of the vertex size (e.g., 5 for
+  // VERTEX_PT).
+  repeated float vertex_buffer = 3;
+  // Index buffer size is a multiple of the primitive size (e.g., 3 for
+  // TRIANGLE).
+  repeated uint32 index_buffer = 4;
+}