talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: d073f8e21be2fcc0e503cb97c6695078b6b75310
This commit is contained in:
MediaPipe Team
2021-02-27 03:30:05 -05:00
committed by chuoling
parent 39309bedba
commit 350fbb2100
755 changed files with 16391 additions and 11075 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/modules/face_detection/BUILD
+17 -13
View File
@@ -26,12 +26,10 @@ mediapipe_simple_subgraph(
graph = "face_detection_front_by_roi_cpu.pbtxt",
register_as = "FaceDetectionFrontByRoiCpu",
deps = [
":face_detection_front_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/util:detection_projection_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
@@ -40,12 +38,10 @@ mediapipe_simple_subgraph(
graph = "face_detection_front_by_roi_gpu.pbtxt",
register_as = "FaceDetectionFrontByRoiGpu",
deps = [
":face_detection_front_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/util:detection_projection_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
@@ -54,12 +50,10 @@ mediapipe_simple_subgraph(
graph = "face_detection_front_cpu.pbtxt",
register_as = "FaceDetectionFrontCpu",
deps = [
":face_detection_front_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
@@ -68,11 +62,21 @@ mediapipe_simple_subgraph(
graph = "face_detection_front_gpu.pbtxt",
register_as = "FaceDetectionFrontGpu",
deps = [
":face_detection_front_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_front_common",
graph = "face_detection_front_common.pbtxt",
register_as = "FaceDetectionFrontCommon",
deps = [
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
"//mediapipe/calculators/util:detection_projection_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
],
)
mediapipe/modules/face_detection/face_detection_front_by_roi_cpu.pbtxt
+12 -72
View File
@@ -13,7 +13,7 @@
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFrontCpu"
type: "FaceDetectionFrontByRoiCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
@@ -29,6 +29,14 @@ input_stream: "ROI:roi"
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input CPU image (ImageFrame) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms specified region of image into 128x128 tensor keeping aspect ratio
# (padding tensor if needed).
node {
@@ -66,78 +74,10 @@ node {
}
}
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
# Performs tensor post processing to generate face detections.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 4
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 128
input_size_width: 128
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 8
strides: 16
strides: 16
strides: 16
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
calculator: "FaceDetectionFrontCommon"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 896
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.5
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Projects the detections from input tensor to the corresponding locations on
# the original image (input to the graph).
node {
calculator: "DetectionProjectionCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "PROJECTION_MATRIX:transform_matrix"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection_front_by_roi_gpu.pbtxt
+13 -73
View File
@@ -13,7 +13,7 @@
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFrontGpu"
type: "FaceDetectionFrontByRoiGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
@@ -29,11 +29,19 @@ input_stream: "ROI:roi"
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input GPU image (GpuBuffer) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms specified region of image into 128x128 tensor keeping aspect ratio
# (padding tensor if needed).
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE_GPU:image"
input_stream: "IMAGE:multi_backend_image"
input_stream: "NORM_RECT:roi"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
@@ -66,78 +74,10 @@ node {
}
}
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
# Performs tensor post processing to generate face detections.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 4
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 128
input_size_width: 128
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 8
strides: 16
strides: 16
strides: 16
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
calculator: "FaceDetectionFrontCommon"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 896
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.5
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Projects the detections from input tensor to the corresponding locations on
# the original image (input to the graph).
node {
calculator: "DetectionProjectionCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "PROJECTION_MATRIX:transform_matrix"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection_front_common.pbtxt
+103
View File
@@ -0,0 +1,103 @@
# MediaPipe graph performing common processing to detect faces, currently
# consisting of tensor post processing.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFrontCommon"
# input_stream: "TENSORS:detection_tensors"
# input_stream: "MATRIX:transform_matrix"
# output_stream: "DETECTIONS:detections"
# }
type: "FaceDetectionFrontCommon"
# Detection tensors. (std::vector<Tensor>)
input_stream: "TENSORS:detection_tensors"
# A 4x4 row-major-order matrix that maps a point represented in the detection
# tensors to a desired coordinate system, e.g., in the original input image
# before scaling/cropping. (std::array<float, 16>)
input_stream: "MATRIX:transform_matrix"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 4
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 128
input_size_width: 128
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 8
strides: 16
strides: 16
strides: 16
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 896
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.5
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Projects the detections from input tensor to the corresponding locations on
# the original image (input to the graph).
node {
calculator: "DetectionProjectionCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "PROJECTION_MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection_front_cpu.pbtxt
+13 -75
View File
@@ -24,14 +24,22 @@ input_stream: "IMAGE:image"
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input CPU image (ImageFrame) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:image"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
@@ -61,80 +69,10 @@ node {
}
}
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
# Performs tensor post processing to generate face detections.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 4
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 128
input_size_width: 128
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 8
strides: 16
strides: 16
strides: 16
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
calculator: "FaceDetectionFrontCommon"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 896
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.5
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
# letterboxed image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (the
# input image to the graph before image transformation).
node {
calculator: "DetectionLetterboxRemovalCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt
+13 -75
View File
@@ -24,14 +24,22 @@ input_stream: "IMAGE:image"
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input GPU image (GpuBuffer) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE_GPU:image"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
@@ -61,80 +69,10 @@ node {
}
}
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
# Performs tensor post processing to generate face detections.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 4
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 128
input_size_width: 128
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 8
strides: 16
strides: 16
strides: 16
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
calculator: "FaceDetectionFrontCommon"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 896
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.5
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
# letterboxed image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (the
# input image to the graph before image transformation).
node {
calculator: "DetectionLetterboxRemovalCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_geometry/BUILD
+21
View File
@@ -28,6 +28,27 @@ mediapipe_simple_subgraph(
],
)
mediapipe_simple_subgraph(
name = "face_geometry_from_detection",
graph = "face_geometry_from_detection.pbtxt",
register_as = "FaceGeometryFromDetection",
deps = [
":geometry_pipeline_calculator",
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/util:detection_to_landmarks_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_geometry_from_landmarks",
graph = "face_geometry_from_landmarks.pbtxt",
register_as = "FaceGeometryFromLandmarks",
deps = [
":geometry_pipeline_calculator",
],
)
mediapipe_proto_library(
name = "effect_renderer_calculator_proto",
srcs = ["effect_renderer_calculator.proto"],
mediapipe/modules/face_geometry/README.md
+3 -1
View File
@@ -15,4 +15,6 @@ Calculators|Details
Subgraphs|Details
:--- | :---
[`FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry.pbtxt)| Extracts face geometry from landmarks for multiple faces.
[`FaceGeometryFromDetection`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_from_detection.pbtxt)| Extracts geometry from face detection for multiple faces.
[`FaceGeometryFromLandmarks`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_from_landmarks.pbtxt)| Extracts geometry from face landmarks for multiple faces.
[`FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry.pbtxt)| Extracts geometry from face landmarks for multiple faces. Deprecated, please use `FaceGeometryFromLandmarks` in the new code.
mediapipe/modules/face_geometry/data/BUILD
+23 -1
View File
@@ -18,9 +18,31 @@ licenses(["notice"])
package(default_visibility = ["//visibility:public"])
encode_binary_proto(
name = "geometry_pipeline_metadata_detection",
input = "geometry_pipeline_metadata_detection.pbtxt",
message_type = "mediapipe.face_geometry.GeometryPipelineMetadata",
output = "geometry_pipeline_metadata_detection.binarypb",
deps = [
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_proto",
],
)
encode_binary_proto(
name = "geometry_pipeline_metadata_landmarks",
input = "geometry_pipeline_metadata_landmarks.pbtxt",
message_type = "mediapipe.face_geometry.GeometryPipelineMetadata",
output = "geometry_pipeline_metadata_landmarks.binarypb",
deps = [
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_proto",
],
)
# For backward-compatibility reasons, generate `geometry_pipeline_metadata.binarypb` from
# the `geometry_pipeline_metadata_landmarks.pbtxt` definition.
encode_binary_proto(
name = "geometry_pipeline_metadata",
input = "geometry_pipeline_metadata.pbtxt",
input = "geometry_pipeline_metadata_landmarks.pbtxt",
message_type = "mediapipe.face_geometry.GeometryPipelineMetadata",
output = "geometry_pipeline_metadata.binarypb",
deps = [
mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_detection.pbtxt
+78
View File
@@ -0,0 +1,78 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
input_source: FACE_DETECTION_PIPELINE
procrustes_landmark_basis { landmark_id: 0 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 1 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 2 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 3 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 4 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 5 weight: 1.0 }
# NOTE: the triangular topology of the face meshes is only useful when derived
# from the 468 face landmarks, not from the 6 face detection landmarks
# (keypoints). The former don't cover the entire face and this mesh is
# defined here only to comply with the API. It should be considered as
# a placeholder and/or for debugging purposes.
#
# Use the face geometry derived from the face detection landmarks
# (keypoints) for the face pose transformation matrix, not the mesh.
canonical_mesh: {
vertex_type: VERTEX_PT
primitive_type: TRIANGLE
vertex_buffer: -3.1511454582214355
vertex_buffer: 2.6246179342269897
vertex_buffer: 3.4656630754470825
vertex_buffer: 0.349575996398926
vertex_buffer: 0.38137748837470997
vertex_buffer: 3.1511454582214355
vertex_buffer: 2.6246179342269897
vertex_buffer: 3.4656630754470825
vertex_buffer: 0.650443494319916
vertex_buffer: 0.38137999176979054
vertex_buffer: 0.0
vertex_buffer: -1.126865029335022
vertex_buffer: 7.475604057312012
vertex_buffer: 0.500025987625122
vertex_buffer: 0.547487020492554
vertex_buffer: 0.0
vertex_buffer: -4.304508209228516
vertex_buffer: 4.162498950958252
vertex_buffer: 0.499989986419678
vertex_buffer: 0.694203019142151
vertex_buffer: -7.664182186126709
vertex_buffer: 0.673132002353668
vertex_buffer: -2.435867071151733
vertex_buffer: 0.007561000064015
vertex_buffer: 0.480777025222778
vertex_buffer: 7.664182186126709
vertex_buffer: 0.673132002353668
vertex_buffer: -2.435867071151733
vertex_buffer: 0.992439985275269
vertex_buffer: 0.480777025222778
index_buffer: 0
index_buffer: 1
index_buffer: 2
index_buffer: 1
index_buffer: 5
index_buffer: 2
index_buffer: 4
index_buffer: 0
index_buffer: 2
index_buffer: 4
index_buffer: 2
index_buffer: 3
index_buffer: 2
index_buffer: 5
index_buffer: 3
}
mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.pbtxt → mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_landmarks.pbtxt
+1
View File
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
input_source: FACE_LANDMARK_PIPELINE
procrustes_landmark_basis { landmark_id: 4 weight: 0.070909939706326 }
procrustes_landmark_basis { landmark_id: 6 weight: 0.032100144773722 }
procrustes_landmark_basis { landmark_id: 10 weight: 0.008446550928056 }
mediapipe/modules/face_geometry/effect_renderer_calculator.cc
+11 -11
View File
@@ -85,7 +85,7 @@ static constexpr char kMultiFaceGeometryTag[] = "MULTI_FACE_GEOMETRY";
//
class EffectRendererCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc) {
static absl::Status GetContract(CalculatorContract* cc) {
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc))
<< "Failed to update contract for the GPU helper!";
@@ -101,12 +101,12 @@ class EffectRendererCalculator : public CalculatorBase {
return mediapipe::GlCalculatorHelper::UpdateContract(cc);
}
mediapipe::Status Open(CalculatorContext* cc) override {
absl::Status Open(CalculatorContext* cc) override {
cc->SetOffset(mediapipe::TimestampDiff(0));
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc))
<< "Failed to open the GPU helper!";
return gpu_helper_.RunInGlContext([&]() -> mediapipe::Status {
return gpu_helper_.RunInGlContext([&]() -> absl::Status {
const auto& options =
cc->Options<FaceGeometryEffectRendererCalculatorOptions>();
@@ -136,19 +136,19 @@ class EffectRendererCalculator : public CalculatorBase {
std::move(effect_texture)),
_ << "Failed to create the effect renderer!");
return mediapipe::OkStatus();
return absl::OkStatus();
});
}
mediapipe::Status Process(CalculatorContext* cc) override {
absl::Status Process(CalculatorContext* cc) override {
// The `IMAGE_GPU` stream is required to have a non-empty packet. In case
// this requirement is not met, there's nothing to be processed at the
// current timestamp.
if (cc->Inputs().Tag(kImageGpuTag).IsEmpty()) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
return gpu_helper_.RunInGlContext([this, cc]() -> mediapipe::Status {
return gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
const auto& input_gpu_buffer =
cc->Inputs().Tag(kImageGpuTag).Get<GpuBuffer>();
@@ -191,7 +191,7 @@ class EffectRendererCalculator : public CalculatorBase {
output_gl_texture.Release();
input_gl_texture.Release();
return mediapipe::OkStatus();
return absl::OkStatus();
});
}
@@ -200,7 +200,7 @@ class EffectRendererCalculator : public CalculatorBase {
}
private:
static mediapipe::StatusOr<ImageFrame> ReadTextureFromFile(
static absl::StatusOr<ImageFrame> ReadTextureFromFile(
const std::string& texture_path) {
ASSIGN_OR_RETURN(std::string texture_blob,
ReadContentBlobFromFile(texture_path),
@@ -244,7 +244,7 @@ class EffectRendererCalculator : public CalculatorBase {
return output_image_frame;
}
static mediapipe::StatusOr<face_geometry::Mesh3d> ReadMesh3dFromFile(
static absl::StatusOr<face_geometry::Mesh3d> ReadMesh3dFromFile(
const std::string& mesh_3d_path) {
ASSIGN_OR_RETURN(std::string mesh_3d_blob,
ReadContentBlobFromFile(mesh_3d_path),
@@ -257,7 +257,7 @@ class EffectRendererCalculator : public CalculatorBase {
return mesh_3d;
}
static mediapipe::StatusOr<std::string> ReadContentBlobFromFile(
static absl::StatusOr<std::string> ReadContentBlobFromFile(
const std::string& unresolved_path) {
ASSIGN_OR_RETURN(std::string resolved_path,
mediapipe::PathToResourceAsFile(unresolved_path),
mediapipe/modules/face_geometry/env_generator_calculator.cc
+8 -8
View File
@@ -40,14 +40,14 @@ static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
//
class EnvGeneratorCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc) {
static absl::Status GetContract(CalculatorContract* cc) {
cc->OutputSidePackets()
.Tag(kEnvironmentTag)
.Set<face_geometry::Environment>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Open(CalculatorContext* cc) override {
absl::Status Open(CalculatorContext* cc) override {
cc->SetOffset(mediapipe::TimestampDiff(0));
const face_geometry::Environment& environment =
@@ -60,15 +60,15 @@ class EnvGeneratorCalculator : public CalculatorBase {
.Tag(kEnvironmentTag)
.Set(mediapipe::MakePacket<face_geometry::Environment>(environment));
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Process(CalculatorContext* cc) override {
return mediapipe::OkStatus();
absl::Status Process(CalculatorContext* cc) override {
return absl::OkStatus();
}
mediapipe::Status Close(CalculatorContext* cc) override {
return mediapipe::OkStatus();
absl::Status Close(CalculatorContext* cc) override {
return absl::OkStatus();
}
};
mediapipe/modules/face_geometry/face_geometry.pbtxt
+6 -10
View File
@@ -1,17 +1,12 @@
# MediaPipe graph to extract face geometry from landmarks for multiple faces.
# MediaPipe graph to extract geometry from face landmarks for multiple faces.
#
# It is required that "geometry_pipeline_metadata.binarypb" is available at
# "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.binarypb"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceGeometry"
# input_stream: "IMAGE_SIZE:image_size"
# input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
# input_side_packet: "ENVIRONMENT:environment"
# output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# }
# This is a deprecated subgraph kept for backward-compatibility reasons. Please,
# be explicit and use the `FaceGeometryFromLandmarks` subgraph in the new code
# to enable the same runtime behaviour.
type: "FaceGeometry"
@@ -37,7 +32,8 @@ input_side_packet: "ENVIRONMENT:environment"
# (std::vector<face_geometry::FaceGeometry>)
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# Extracts face geometry for multiple faces from a vector of landmark lists.
# Extracts face geometry for multiple faces from a vector of face landmark
# lists.
node {
calculator: "FaceGeometryPipelineCalculator"
input_side_packet: "ENVIRONMENT:environment"
mediapipe/modules/face_geometry/face_geometry_from_detection.pbtxt
+87
View File
@@ -0,0 +1,87 @@
# MediaPipe graph to extract geometry from face detection for multiple faces.
#
# It is required that "geometry_pipeline_metadata_detection.binarypb" is
# available at
# "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_detection.binarypb"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceGeometryFromDetection"
# input_stream: "IMAGE_SIZE:image_size"
# input_stream: "MULTI_FACE_DETECTION:multi_face_detection"
# input_side_packet: "ENVIRONMENT:environment"
# output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# }
type: "FaceGeometryFromDetection"
# The size of the input frame. The first element of the pair is the frame width;
# the other one is the frame height.
#
# The face landmarks should have been detected on a frame with the same
# ratio. If used as-is, the resulting face geometry visualization should be
# happening on a frame with the same ratio as well.
#
# (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Collection of detected/predicted faces, each represented as a detection.
# (std::vector<DETECTION>)
input_stream: "MULTI_FACE_DETECTION:multi_face_detection"
# Environment that describes the current virtual scene.
# (face_geometry::Environment)
input_side_packet: "ENVIRONMENT:environment"
# A list of geometry data for each detected face.
# (std::vector<face_geometry::FaceGeometry>)
#
# NOTE: the triangular topology of the face meshes is only useful when derived
# from the 468 face landmarks, not from the 6 face detection landmarks
# (keypoints). The former don't cover the entire face and this mesh is
# defined here only to comply with the API. It should be considered as
# a placeholder and/or for debugging purposes.
#
# Use the face geometry derived from the face detection landmarks
# (keypoints) for the face pose transformation matrix, not the mesh.
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# Begin iterating over a vector of the face detections.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:multi_face_detection"
output_stream: "ITEM:face_detection"
output_stream: "BATCH_END:detection_timestamp"
}
# Extracts face detection keypoints as a normalized landmarks.
node {
calculator: "DetectionToLandmarksCalculator"
input_stream: "DETECTION:face_detection"
output_stream: "LANDMARKS:face_landmarks"
}
# End iterating over a vector of the face detections and receive a vector of
# face landmark lists as a result.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:face_landmarks"
input_stream: "BATCH_END:detection_timestamp"
output_stream: "ITERABLE:multi_face_landmarks"
}
# Extracts face geometry for multiple faces from a vector of face detection
# landmark lists.
node {
calculator: "FaceGeometryPipelineCalculator"
input_side_packet: "ENVIRONMENT:environment"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
options: {
[mediapipe.FaceGeometryPipelineCalculatorOptions.ext] {
metadata_path: "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_detection.binarypb"
}
}
}
mediapipe/modules/face_geometry/face_geometry_from_landmarks.pbtxt
+54
View File
@@ -0,0 +1,54 @@
# MediaPipe graph to extract geometry from face landmarks for multiple faces.
#
# It is required that "geometry_pipeline_metadata_from_landmark.binarypb" is
# available at
# "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_from_landmarks.binarypb"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceGeometryFromLandmarks"
# input_stream: "IMAGE_SIZE:image_size"
# input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
# input_side_packet: "ENVIRONMENT:environment"
# output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# }
type: "FaceGeometryFromLandmarks"
# The size of the input frame. The first element of the pair is the frame width;
# the other one is the frame height.
#
# The face landmarks should have been detected on a frame with the same
# ratio. If used as-is, the resulting face geometry visualization should be
# happening on a frame with the same ratio as well.
#
# (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Collection of detected/predicted faces, each represented as a list of face
# landmarks. (std::vector<NormalizedLandmarkList>)
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
# Environment that describes the current virtual scene.
# (face_geometry::Environment)
input_side_packet: "ENVIRONMENT:environment"
# A list of geometry data for each detected face.
# (std::vector<face_geometry::FaceGeometry>)
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# Extracts face geometry for multiple faces from a vector of face landmark
# lists.
node {
calculator: "FaceGeometryPipelineCalculator"
input_side_packet: "ENVIRONMENT:environment"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
options: {
[mediapipe.FaceGeometryPipelineCalculatorOptions.ext] {
metadata_path: "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_landmarks.binarypb"
}
}
}
mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc
+11 -11
View File
@@ -72,7 +72,7 @@ static constexpr char kMultiFaceLandmarksTag[] = "MULTI_FACE_LANDMARKS";
//
class GeometryPipelineCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc) {
static absl::Status GetContract(CalculatorContract* cc) {
cc->InputSidePackets()
.Tag(kEnvironmentTag)
.Set<face_geometry::Environment>();
@@ -84,10 +84,10 @@ class GeometryPipelineCalculator : public CalculatorBase {
.Tag(kMultiFaceGeometryTag)
.Set<std::vector<face_geometry::FaceGeometry>>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Open(CalculatorContext* cc) override {
absl::Status Open(CalculatorContext* cc) override {
cc->SetOffset(mediapipe::TimestampDiff(0));
const auto& options = cc->Options<FaceGeometryPipelineCalculatorOptions>();
@@ -114,16 +114,16 @@ class GeometryPipelineCalculator : public CalculatorBase {
face_geometry::CreateGeometryPipeline(environment, metadata),
_ << "Failed to create a geometry pipeline!");
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Process(CalculatorContext* cc) override {
absl::Status Process(CalculatorContext* cc) override {
// Both the `IMAGE_SIZE` and the `MULTI_FACE_LANDMARKS` streams are required
// to have a non-empty packet. In case this requirement is not met, there's
// nothing to be processed at the current timestamp.
if (cc->Inputs().Tag(kImageSizeTag).IsEmpty() ||
cc->Inputs().Tag(kMultiFaceLandmarksTag).IsEmpty()) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
const auto& image_size =
@@ -150,15 +150,15 @@ class GeometryPipelineCalculator : public CalculatorBase {
multi_face_geometry.release())
.At(cc->InputTimestamp()));
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Close(CalculatorContext* cc) override {
return mediapipe::OkStatus();
absl::Status Close(CalculatorContext* cc) override {
return absl::OkStatus();
}
private:
static mediapipe::StatusOr<face_geometry::GeometryPipelineMetadata>
static absl::StatusOr<face_geometry::GeometryPipelineMetadata>
ReadMetadataFromFile(const std::string& metadata_path) {
ASSIGN_OR_RETURN(std::string metadata_blob,
ReadContentBlobFromFile(metadata_path),
@@ -171,7 +171,7 @@ class GeometryPipelineCalculator : public CalculatorBase {
return metadata;
}
static mediapipe::StatusOr<std::string> ReadContentBlobFromFile(
static absl::StatusOr<std::string> ReadContentBlobFromFile(
const std::string& unresolved_path) {
ASSIGN_OR_RETURN(std::string resolved_path,
mediapipe::PathToResourceAsFile(unresolved_path),
mediapipe/modules/face_geometry/libs/effect_renderer.cc
+17 -18
View File
@@ -42,7 +42,7 @@ namespace mediapipe::face_geometry {
namespace {
struct RenderableMesh3d {
static mediapipe::StatusOr<RenderableMesh3d> CreateFromProtoMesh3d(
static absl::StatusOr<RenderableMesh3d> CreateFromProtoMesh3d(
const Mesh3d& proto_mesh_3d) {
Mesh3d::VertexType vertex_type = proto_mesh_3d.vertex_type();
@@ -106,14 +106,14 @@ struct RenderableMesh3d {
class Texture {
public:
static mediapipe::StatusOr<std::unique_ptr<Texture>> WrapExternalTexture(
static absl::StatusOr<std::unique_ptr<Texture>> WrapExternalTexture(
GLuint handle, GLenum target, int width, int height) {
RET_CHECK(handle) << "External texture must have a non-null handle!";
return absl::WrapUnique(new Texture(handle, target, width, height,
/*is_owned*/ false));
}
static mediapipe::StatusOr<std::unique_ptr<Texture>> CreateFromImageFrame(
static absl::StatusOr<std::unique_ptr<Texture>> CreateFromImageFrame(
const ImageFrame& image_frame) {
RET_CHECK(image_frame.IsAligned(ImageFrame::kGlDefaultAlignmentBoundary))
<< "Image frame memory must be aligned for GL usage!";
@@ -187,7 +187,7 @@ class Texture {
class RenderTarget {
public:
static mediapipe::StatusOr<std::unique_ptr<RenderTarget>> Create() {
static absl::StatusOr<std::unique_ptr<RenderTarget>> Create() {
GLuint framebuffer_handle;
glGenFramebuffers(1, &framebuffer_handle);
RET_CHECK(framebuffer_handle)
@@ -205,7 +205,7 @@ class RenderTarget {
}
}
mediapipe::Status SetColorbuffer(const Texture& colorbuffer_texture) {
absl::Status SetColorbuffer(const Texture& colorbuffer_texture) {
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_handle_);
glViewport(0, 0, colorbuffer_texture.width(), colorbuffer_texture.height());
@@ -245,7 +245,7 @@ class RenderTarget {
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glFlush();
return mediapipe::OkStatus();
return absl::OkStatus();
}
void Bind() const {
@@ -288,7 +288,7 @@ class Renderer {
public:
enum class RenderMode { OPAQUE, OVERDRAW, OCCLUSION };
static mediapipe::StatusOr<std::unique_ptr<Renderer>> Create() {
static absl::StatusOr<std::unique_ptr<Renderer>> Create() {
static const GLint kAttrLocation[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
ATTRIB_TEXTURE_POSITION,
@@ -346,12 +346,11 @@ class Renderer {
~Renderer() { glDeleteProgram(program_handle_); }
mediapipe::Status Render(const RenderTarget& render_target,
const Texture& texture,
const RenderableMesh3d& mesh_3d,
const std::array<float, 16>& projection_mat,
const std::array<float, 16>& model_mat,
RenderMode render_mode) const {
absl::Status Render(const RenderTarget& render_target, const Texture& texture,
const RenderableMesh3d& mesh_3d,
const std::array<float, 16>& projection_mat,
const std::array<float, 16>& model_mat,
RenderMode render_mode) const {
glUseProgram(program_handle_);
// Set up the GL state.
glEnable(GL_BLEND);
@@ -413,7 +412,7 @@ class Renderer {
glUseProgram(0);
glFlush();
return mediapipe::OkStatus();
return absl::OkStatus();
}
private:
@@ -451,7 +450,7 @@ class EffectRendererImpl : public EffectRenderer {
effect_texture_(std::move(effect_texture)),
identity_matrix_(Create4x4IdentityMatrix()) {}
mediapipe::Status RenderEffect(
absl::Status RenderEffect(
const std::vector<FaceGeometry>& multi_face_geometry,
int frame_width, //
int frame_height, //
@@ -567,7 +566,7 @@ class EffectRendererImpl : public EffectRenderer {
// At this point in the code, the destination texture must contain the
// correctly renderer effect, so we should just return.
return mediapipe::OkStatus();
return absl::OkStatus();
}
private:
@@ -606,7 +605,7 @@ class EffectRendererImpl : public EffectRenderer {
0.f, 0.f, 0.f, 1.f};
}
static mediapipe::StatusOr<std::array<float, 16>>
static absl::StatusOr<std::array<float, 16>>
Convert4x4MatrixDataToArrayFormat(const MatrixData& matrix_data) {
RET_CHECK(matrix_data.rows() == 4 && //
matrix_data.cols() == 4 && //
@@ -689,7 +688,7 @@ ImageFrame CreateEmptyColorTexture() {
} // namespace
mediapipe::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
absl::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
const Environment& environment, //
const absl::optional<Mesh3d>& effect_mesh_3d, //
ImageFrame&& effect_texture) {
mediapipe/modules/face_geometry/libs/effect_renderer.h
+2 -2
View File
@@ -49,7 +49,7 @@ class EffectRenderer {
// reference existing OpenGL textures in the current context. They should also
// reference different textures as the in-place effect rendering is not yet
// supported.
virtual mediapipe::Status RenderEffect(
virtual absl::Status RenderEffect(
const std::vector<FaceGeometry>& multi_face_geometry,
int frame_width, //
int frame_height, //
@@ -82,7 +82,7 @@ class EffectRenderer {
//
// `effect_texture` must have positive dimensions. Its format must be either
// `SRGB` or `SRGBA`. Its memory must be aligned for GL usage.
mediapipe::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
absl::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
const Environment& environment, //
const absl::optional<Mesh3d>& effect_mesh_3d, //
ImageFrame&& effect_texture);
mediapipe/modules/face_geometry/libs/geometry_pipeline.cc
+54 -22
View File
@@ -73,10 +73,12 @@ class ScreenToMetricSpaceConverter {
public:
ScreenToMetricSpaceConverter(
OriginPointLocation origin_point_location, //
InputSource input_source, //
Eigen::Matrix3Xf&& canonical_metric_landmarks, //
Eigen::VectorXf&& landmark_weights, //
std::unique_ptr<ProcrustesSolver> procrustes_solver)
: origin_point_location_(origin_point_location),
input_source_(input_source),
canonical_metric_landmarks_(std::move(canonical_metric_landmarks)),
landmark_weights_(std::move(landmark_weights)),
procrustes_solver_(std::move(procrustes_solver)) {}
@@ -118,11 +120,10 @@ class ScreenToMetricSpaceConverter {
//
// To keep the logic correct, the landmark set handedness is changed any
// time the screen-to-metric semantic barrier is passed.
mediapipe::Status Convert(
const NormalizedLandmarkList& screen_landmark_list, //
const PerspectiveCameraFrustum& pcf, //
LandmarkList& metric_landmark_list, //
Eigen::Matrix4f& pose_transform_mat) const {
absl::Status Convert(const NormalizedLandmarkList& screen_landmark_list, //
const PerspectiveCameraFrustum& pcf, //
LandmarkList& metric_landmark_list, //
Eigen::Matrix4f& pose_transform_mat) const {
RET_CHECK_EQ(screen_landmark_list.landmark_size(),
canonical_metric_landmarks_.cols())
<< "The number of landmarks doesn't match the number passed upon "
@@ -151,12 +152,27 @@ class ScreenToMetricSpaceConverter {
intermediate_landmarks);
UnprojectXY(pcf, intermediate_landmarks);
ChangeHandedness(intermediate_landmarks);
// For face detection input landmarks, re-write Z-coord from the canonical
// landmarks.
if (input_source_ == InputSource::FACE_DETECTION_PIPELINE) {
Eigen::Matrix4f intermediate_pose_transform_mat;
MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
canonical_metric_landmarks_, intermediate_landmarks,
landmark_weights_, intermediate_pose_transform_mat))
<< "Failed to estimate pose transform matrix!";
intermediate_landmarks.row(2) =
(intermediate_pose_transform_mat *
canonical_metric_landmarks_.colwise().homogeneous())
.row(2);
}
ASSIGN_OR_RETURN(const float second_iteration_scale,
EstimateScale(intermediate_landmarks),
_ << "Failed to estimate second iteration scale!");
// Use the total scale to unproject the screen landmarks.
float total_scale = first_iteration_scale * second_iteration_scale;
const float total_scale = first_iteration_scale * second_iteration_scale;
MoveAndRescaleZ(pcf, depth_offset, total_scale, screen_landmarks);
UnprojectXY(pcf, screen_landmarks);
ChangeHandedness(screen_landmarks);
@@ -169,18 +185,30 @@ class ScreenToMetricSpaceConverter {
pose_transform_mat))
<< "Failed to estimate pose transform matrix!";
// Multiply each of the metric landmarks by the inverse pose transformation
// matrix to align the runtime metric face landmarks with the canonical
// metric face landmarks.
Eigen::Matrix4f inv_pose_transform_mat = pose_transform_mat.inverse();
auto inv_pose_rotation = inv_pose_transform_mat.leftCols(3).topRows(3);
auto inv_pose_translation = inv_pose_transform_mat.col(3).topRows(3);
metric_landmarks =
(inv_pose_rotation * metric_landmarks).colwise() + inv_pose_translation;
// For face detection input landmarks, re-write Z-coord from the canonical
// landmarks and run the pose transform estimation again.
if (input_source_ == InputSource::FACE_DETECTION_PIPELINE) {
metric_landmarks.row(2) =
(pose_transform_mat *
canonical_metric_landmarks_.colwise().homogeneous())
.row(2);
MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
canonical_metric_landmarks_, metric_landmarks, landmark_weights_,
pose_transform_mat))
<< "Failed to estimate pose transform matrix!";
}
// Multiply each of the metric landmarks by the inverse pose
// transformation matrix to align the runtime metric face landmarks with
// the canonical metric face landmarks.
metric_landmarks = (pose_transform_mat.inverse() *
metric_landmarks.colwise().homogeneous())
.topRows(3);
ConvertEigenMatrixToLandmarkList(metric_landmarks, metric_landmark_list);
return mediapipe::OkStatus();
return absl::OkStatus();
}
private:
@@ -200,7 +228,7 @@ class ScreenToMetricSpaceConverter {
landmarks.colwise() += Eigen::Vector3f(x_translation, y_translation, 0.f);
}
mediapipe::StatusOr<float> EstimateScale(Eigen::Matrix3Xf& landmarks) const {
absl::StatusOr<float> EstimateScale(Eigen::Matrix3Xf& landmarks) const {
Eigen::Matrix4f transform_mat;
MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
canonical_metric_landmarks_, landmarks, landmark_weights_,
@@ -253,7 +281,8 @@ class ScreenToMetricSpaceConverter {
}
}
OriginPointLocation origin_point_location_;
const OriginPointLocation origin_point_location_;
const InputSource input_source_;
Eigen::Matrix3Xf canonical_metric_landmarks_;
Eigen::VectorXf landmark_weights_;
@@ -277,7 +306,7 @@ class GeometryPipelineImpl : public GeometryPipeline {
canonical_mesh_vertex_position_offset),
space_converter_(std::move(space_converter)) {}
mediapipe::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
absl::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
const std::vector<NormalizedLandmarkList>& multi_face_landmarks,
int frame_width, int frame_height) const override {
MP_RETURN_IF_ERROR(ValidateFrameDimensions(frame_width, frame_height))
@@ -301,8 +330,8 @@ class GeometryPipelineImpl : public GeometryPipeline {
continue;
}
// Convert the screen landmarks into the metric landmarks and
// get the pose transformation matrix.
// Convert the screen landmarks into the metric landmarks and get the pose
// transformation matrix.
LandmarkList metric_face_landmarks;
Eigen::Matrix4f pose_transform_mat;
MP_RETURN_IF_ERROR(space_converter_->Convert(screen_face_landmarks, pcf,
@@ -370,7 +399,7 @@ class GeometryPipelineImpl : public GeometryPipeline {
} // namespace
mediapipe::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
absl::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
const Environment& environment, const GeometryPipelineMetadata& metadata) {
MP_RETURN_IF_ERROR(ValidateEnvironment(environment))
<< "Invalid environment!";
@@ -392,7 +421,7 @@ mediapipe::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
uint32_t canonical_mesh_vertex_position_offset =
GetVertexComponentOffset(canonical_mesh.vertex_type(),
VertexComponent::POSITION)
.ValueOrDie();
.value();
// Put the Procrustes landmark basis into Eigen matrices for an easier access.
Eigen::Matrix3Xf canonical_metric_landmarks =
@@ -424,6 +453,9 @@ mediapipe::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
canonical_mesh_vertex_position_offset,
absl::make_unique<ScreenToMetricSpaceConverter>(
environment.origin_point_location(),
metadata.input_source() == InputSource::DEFAULT
? InputSource::FACE_LANDMARK_PIPELINE
: metadata.input_source(),
std::move(canonical_metric_landmarks),
std::move(landmark_weights),
CreateFloatPrecisionProcrustesSolver()));
mediapipe/modules/face_geometry/libs/geometry_pipeline.h
+2 -2
View File
@@ -47,7 +47,7 @@ class GeometryPipeline {
// geometry pipeline metadata).
//
// Both `frame_width` and `frame_height` must be positive.
virtual mediapipe::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
virtual absl::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
const std::vector<NormalizedLandmarkList>& multi_face_landmarks,
int frame_width, int frame_height) const = 0;
};
@@ -59,7 +59,7 @@ class GeometryPipeline {
//
// Canonical face mesh (defined as a part of `metadata`) must have the
// `POSITION` and the `TEX_COORD` vertex components.
mediapipe::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
absl::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
const Environment& environment, const GeometryPipelineMetadata& metadata);
} // namespace mediapipe::face_geometry
mediapipe/modules/face_geometry/libs/mesh_3d_utils.cc
+2 -2
View File
@@ -78,7 +78,7 @@ bool HasVertexComponent(Mesh3d::VertexType vertex_type,
}
}
mediapipe::StatusOr<uint32_t> GetVertexComponentOffset(
absl::StatusOr<uint32_t> GetVertexComponentOffset(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component) {
RET_CHECK(HasVertexComponentVertexPT(vertex_component))
<< "A given vertex type doesn't have the requested component!";
@@ -89,7 +89,7 @@ mediapipe::StatusOr<uint32_t> GetVertexComponentOffset(
}
}
mediapipe::StatusOr<uint32_t> GetVertexComponentSize(
absl::StatusOr<uint32_t> GetVertexComponentSize(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component) {
RET_CHECK(HasVertexComponentVertexPT(vertex_component))
<< "A given vertex type doesn't have the requested component!";
mediapipe/modules/face_geometry/libs/mesh_3d_utils.h
+2 -2
View File
@@ -36,14 +36,14 @@ bool HasVertexComponent(Mesh3d::VertexType vertex_type,
//
// Returns an error status if a given vertex type doesn't have the requested
// component.
mediapipe::StatusOr<uint32_t> GetVertexComponentOffset(
absl::StatusOr<uint32_t> GetVertexComponentOffset(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component);
// Computes the vertex component size.
//
// Returns an error status if a given vertex type doesn't have the requested
// component.
mediapipe::StatusOr<uint32_t> GetVertexComponentSize(
absl::StatusOr<uint32_t> GetVertexComponentSize(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component);
} // namespace mediapipe::face_geometry
mediapipe/modules/face_geometry/libs/procrustes_solver.cc
+11 -11
View File
@@ -32,7 +32,7 @@ class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
public:
FloatPrecisionProcrustesSolver() = default;
mediapipe::Status SolveWeightedOrthogonalProblem(
absl::Status SolveWeightedOrthogonalProblem(
const Eigen::Matrix3Xf& source_points, //
const Eigen::Matrix3Xf& target_points, //
const Eigen::VectorXf& point_weights,
@@ -52,13 +52,13 @@ class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
source_points, target_points, sqrt_weights, transform_mat))
<< "Failed to solve the WEOP problem!";
return mediapipe::OkStatus();
return absl::OkStatus();
}
private:
static constexpr float kAbsoluteErrorEps = 1e-9f;
static mediapipe::Status ValidateInputPoints(
static absl::Status ValidateInputPoints(
const Eigen::Matrix3Xf& source_points,
const Eigen::Matrix3Xf& target_points) {
RET_CHECK_GT(source_points.cols(), 0)
@@ -67,10 +67,10 @@ class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
RET_CHECK_EQ(source_points.cols(), target_points.cols())
<< "The number of source and target points must be equal!";
return mediapipe::OkStatus();
return absl::OkStatus();
}
static mediapipe::Status ValidatePointWeights(
static absl::Status ValidatePointWeights(
int num_points, const Eigen::VectorXf& point_weights) {
RET_CHECK_GT(point_weights.size(), 0)
<< "The number of point weights must be positive!";
@@ -89,7 +89,7 @@ class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
RET_CHECK_GT(total_weight, kAbsoluteErrorEps)
<< "The total point weight is too small!";
return mediapipe::OkStatus();
return absl::OkStatus();
}
static Eigen::VectorXf ExtractSquareRoot(
@@ -139,7 +139,7 @@ class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
// Note: the output `transform_mat` argument is used instead of `StatusOr<>`
// return type in order to avoid Eigen memory alignment issues. Details:
// https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
static mediapipe::Status InternalSolveWeightedOrthogonalProblem(
static absl::Status InternalSolveWeightedOrthogonalProblem(
const Eigen::Matrix3Xf& sources, const Eigen::Matrix3Xf& targets,
const Eigen::VectorXf& sqrt_weights, Eigen::Matrix4f& transform_mat) {
// tranposed(A_w).
@@ -195,7 +195,7 @@ class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
transform_mat = CombineTransformMatrix(rotation_and_scale, translation);
return mediapipe::OkStatus();
return absl::OkStatus();
}
// `design_matrix` is a transposed LHS of (51) in the paper.
@@ -203,7 +203,7 @@ class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
// Note: the output `rotation` argument is used instead of `StatusOr<>`
// return type in order to avoid Eigen memory alignment issues. Details:
// https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
static mediapipe::Status ComputeOptimalRotation(
static absl::Status ComputeOptimalRotation(
const Eigen::Matrix3f& design_matrix, Eigen::Matrix3f& rotation) {
RET_CHECK_GT(design_matrix.norm(), kAbsoluteErrorEps)
<< "Design matrix norm is too small!";
@@ -228,10 +228,10 @@ class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
// Transposed (52) from the paper.
rotation = postrotation * prerotation;
return mediapipe::OkStatus();
return absl::OkStatus();
}
static mediapipe::StatusOr<float> ComputeOptimalScale(
static absl::StatusOr<float> ComputeOptimalScale(
const Eigen::Matrix3Xf& centered_weighted_sources,
const Eigen::Matrix3Xf& weighted_sources,
const Eigen::Matrix3Xf& weighted_targets,
mediapipe/modules/face_geometry/libs/procrustes_solver.h
+1 -1
View File
@@ -56,7 +56,7 @@ class ProcrustesSolver {
// Note: the output `transform_mat` argument is used instead of `StatusOr<>`
// return type in order to avoid Eigen memory alignment issues. Details:
// https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
virtual mediapipe::Status SolveWeightedOrthogonalProblem(
virtual absl::Status SolveWeightedOrthogonalProblem(
const Eigen::Matrix3Xf& source_points, //
const Eigen::Matrix3Xf& target_points, //
const Eigen::VectorXf& point_weights, //
mediapipe/modules/face_geometry/libs/validation_utils.cc
+12 -12
View File
@@ -28,7 +28,7 @@
namespace mediapipe::face_geometry {
mediapipe::Status ValidatePerspectiveCamera(
absl::Status ValidatePerspectiveCamera(
const PerspectiveCamera& perspective_camera) {
static constexpr float kAbsoluteErrorEps = 1e-9f;
@@ -46,18 +46,18 @@ mediapipe::Status ValidatePerspectiveCamera(
180.f)
<< "Vertical FOV must be less than 180 degrees with a margin of 10^{-9}";
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status ValidateEnvironment(const Environment& environment) {
absl::Status ValidateEnvironment(const Environment& environment) {
MP_RETURN_IF_ERROR(
ValidatePerspectiveCamera(environment.perspective_camera()))
<< "Invalid perspective camera!";
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status ValidateMesh3d(const Mesh3d& mesh_3d) {
absl::Status ValidateMesh3d(const Mesh3d& mesh_3d) {
const std::size_t vertex_size = GetVertexSize(mesh_3d.vertex_type());
const std::size_t primitive_type = GetPrimitiveSize(mesh_3d.primitive_type());
@@ -73,10 +73,10 @@ mediapipe::Status ValidateMesh3d(const Mesh3d& mesh_3d) {
<< "All mesh indices must refer to an existing vertex!";
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status ValidateFaceGeometry(const FaceGeometry& face_geometry) {
absl::Status ValidateFaceGeometry(const FaceGeometry& face_geometry) {
MP_RETURN_IF_ERROR(ValidateMesh3d(face_geometry.mesh())) << "Invalid mesh!";
static constexpr char kInvalid4x4MatrixMessage[] =
@@ -89,10 +89,10 @@ mediapipe::Status ValidateFaceGeometry(const FaceGeometry& face_geometry) {
RET_CHECK_EQ(pose_transform_matrix.packed_data_size(), 16)
<< kInvalid4x4MatrixMessage;
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status ValidateGeometryPipelineMetadata(
absl::Status ValidateGeometryPipelineMetadata(
const GeometryPipelineMetadata& metadata) {
MP_RETURN_IF_ERROR(ValidateMesh3d(metadata.canonical_mesh()))
<< "Invalid canonical mesh!";
@@ -113,14 +113,14 @@ mediapipe::Status ValidateGeometryPipelineMetadata(
<< "All Procrustes basis landmarks must have a non-negative weight!";
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status ValidateFrameDimensions(int frame_width, int frame_height) {
absl::Status ValidateFrameDimensions(int frame_width, int frame_height) {
RET_CHECK_GT(frame_width, 0) << "Frame width must be positive!";
RET_CHECK_GT(frame_height, 0) << "Frame height must be positive!";
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace mediapipe::face_geometry
mediapipe/modules/face_geometry/libs/validation_utils.h
+6 -6
View File
@@ -29,26 +29,26 @@ namespace mediapipe::face_geometry {
// Far Z must be greater than Near Z with a margin of `1e-9`.
// Vertical FOV must be in range (0, 180) with a margin of `1e-9` on the range
// edges.
mediapipe::Status ValidatePerspectiveCamera(
absl::Status ValidatePerspectiveCamera(
const PerspectiveCamera& perspective_camera);
// Validates `environment`.
//
// Environment's perspective camera must be valid.
mediapipe::Status ValidateEnvironment(const Environment& environment);
absl::Status ValidateEnvironment(const Environment& environment);
// Validates `mesh_3d`.
//
// Mesh vertex buffer size must a multiple of the vertex size.
// Mesh index buffer size must a multiple of the primitive size.
// All mesh indices must reference an existing mesh vertex.
mediapipe::Status ValidateMesh3d(const Mesh3d& mesh_3d);
absl::Status ValidateMesh3d(const Mesh3d& mesh_3d);
// Validates `face_geometry`.
//
// Face mesh must be valid.
// Face pose transformation matrix must be a 4x4 matrix.
mediapipe::Status ValidateFaceGeometry(const FaceGeometry& face_geometry);
absl::Status ValidateFaceGeometry(const FaceGeometry& face_geometry);
// Validates `metadata`.
//
@@ -57,13 +57,13 @@ mediapipe::Status ValidateFaceGeometry(const FaceGeometry& face_geometry);
// All Procrustes basis indices must reference an existing canonical mesh
// vertex.
// All Procrustes basis landmarks must have a non-negative weight.
mediapipe::Status ValidateGeometryPipelineMetadata(
absl::Status ValidateGeometryPipelineMetadata(
const GeometryPipelineMetadata& metadata);
// Validates frame dimensions.
//
// Both frame width and frame height must be positive.
mediapipe::Status ValidateFrameDimensions(int frame_width, int frame_height);
absl::Status ValidateFrameDimensions(int frame_width, int frame_height);
} // namespace mediapipe::face_geometry
mediapipe/modules/face_geometry/protos/face_geometry.proto
+9
View File
@@ -34,6 +34,15 @@ message FaceGeometry {
// the input face landmarks after (1) being multiplied by the face pose
// transformation matrix and then (2) being projected with a perspective
// camera matrix of the same environment.
//
// NOTE: the triangular topology of the face mesh is only useful when derived
// from the 468 face landmarks, not from the 6 face detection landmarks
// (keypoints). The former don't cover the entire face and this mesh is
// defined here only to comply with the API. It should be considered as
// a placeholder and/or for debugging purposes.
//
// Use the face geometry derived from the face detection landmarks
// (keypoints) for the face pose transformation matrix, not the mesh.
optional Mesh3d mesh = 1;
// Defines a face pose transformation matrix, which provides mapping from
mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.proto
+17
View File
@@ -21,6 +21,12 @@ import "mediapipe/modules/face_geometry/protos/mesh_3d.proto";
option java_package = "com.google.mediapipe.modules.facegeometry";
option java_outer_classname = "GeometryPipelineMetadataProto";
enum InputSource {
DEFAULT = 0; // FACE_LANDMARK_PIPELINE
FACE_LANDMARK_PIPELINE = 1;
FACE_DETECTION_PIPELINE = 2;
}
message WeightedLandmarkRef {
// Defines the landmark ID. References an existing face landmark ID.
optional uint32 landmark_id = 1;
@@ -31,7 +37,18 @@ message WeightedLandmarkRef {
optional float weight = 2;
}
// Next field ID: 4
message GeometryPipelineMetadata {
// Defines the source of the input landmarks to let the underlying geometry
// pipeline to adjust in order to produce the best results.
//
// Face landmark pipeline is expected to produce 3D landmarks with relative Z
// coordinate, which is scaled as the X coordinate assuming the weak
// perspective projection camera model.
//
// Face landmark pipeline is expected to produce 2D landmarks with Z
// coordinate being equal to 0.
optional InputSource input_source = 3;
// Defines a mesh surface for a canonical face. The canonical face mesh vertex
// IDs are the same as the face landmark IDs.
//
mediapipe/modules/hand_landmark/BUILD
+2 -1
View File
@@ -23,6 +23,7 @@ package(default_visibility = ["//visibility:public"])
exports_files([
"hand_landmark.tflite",
"hand_landmark_sparse.tflite",
"handedness.txt",
])
@@ -104,7 +105,7 @@ mediapipe_simple_subgraph(
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/calculators/util:filter_collection_calculator",
"//mediapipe/modules/palm_detection:palm_detection_gpu",
"//mediapipe/modules/palm_detection:palm_detection_cpu",
],
)
mediapipe/modules/hand_landmark/calculators/hand_landmarks_to_rect_calculator.cc
+9 -9
View File
@@ -58,7 +58,7 @@ float ComputeRotation(const NormalizedLandmarkList& landmarks,
return rotation;
}
mediapipe::Status NormalizedLandmarkListToRect(
absl::Status NormalizedLandmarkListToRect(
const NormalizedLandmarkList& landmarks,
const std::pair<int, int>& image_size, NormalizedRect* rect) {
const float rotation = ComputeRotation(landmarks, image_size);
@@ -117,7 +117,7 @@ mediapipe::Status NormalizedLandmarkListToRect(
rect->set_height(height);
rect->set_rotation(rotation);
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace
@@ -130,21 +130,21 @@ mediapipe::Status NormalizedLandmarkListToRect(
// mean of PIP joints at the top.
class HandLandmarksToRectCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc) {
static absl::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs().Tag(kNormRectTag).Set<NormalizedRect>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Open(CalculatorContext* cc) override {
absl::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Process(CalculatorContext* cc) override {
absl::Status Process(CalculatorContext* cc) override {
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
@@ -159,7 +159,7 @@ class HandLandmarksToRectCalculator : public CalculatorBase {
.Tag(kNormRectTag)
.Add(output_rect.release(), cc->InputTimestamp());
return mediapipe::OkStatus();
return absl::OkStatus();
}
};
REGISTER_CALCULATOR(HandLandmarksToRectCalculator);
mediapipe/modules/hand_landmark/hand_landmark_cpu.pbtxt
+1
View File
@@ -52,6 +52,7 @@ node {
model_path: "mediapipe/modules/hand_landmark/hand_landmark.tflite"
delegate { xnnpack {} }
}
#
}
}
mediapipe/modules/hand_landmark/hand_landmark_sparse.tflite
Executable
BIN
View File
Binary file not shown.
mediapipe/modules/holistic_landmark/README.md
+1 -1
View File
@@ -3,4 +3,4 @@
Subgraphs|Details
:--- | :---
[`HolisticLandmarkCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt)| Predicts pose + left/right hand + face landmarks. (CPU input)
[`HolisticLandmarkCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt)| Predicts pose + left/right hand + face landmarks. (GPU input.)
[`HolisticLandmarkGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt)| Predicts pose + left/right hand + face landmarks. (GPU input.)
mediapipe/modules/holistic_landmark/calculators/hand_detections_from_pose_to_rects_calculator.cc
+13 -13
View File
@@ -39,15 +39,15 @@ namespace {} // namespace
class HandDetectionsFromPoseToRectsCalculator
: public DetectionsToRectsCalculator {
public:
::mediapipe::Status Open(CalculatorContext* cc) override;
absl::Status Open(CalculatorContext* cc) override;
private:
::mediapipe::Status DetectionToNormalizedRect(
const Detection& detection, const DetectionSpec& detection_spec,
NormalizedRect* rect) override;
::mediapipe::Status ComputeRotation(const Detection& detection,
const DetectionSpec& detection_spec,
float* rotation) override;
::absl::Status DetectionToNormalizedRect(const Detection& detection,
const DetectionSpec& detection_spec,
NormalizedRect* rect) override;
absl::Status ComputeRotation(const Detection& detection,
const DetectionSpec& detection_spec,
float* rotation) override;
};
REGISTER_CALCULATOR(HandDetectionsFromPoseToRectsCalculator);
@@ -61,7 +61,7 @@ constexpr char kImageSizeTag[] = "IMAGE_SIZE";
} // namespace
::mediapipe::Status HandDetectionsFromPoseToRectsCalculator::Open(
::absl::Status HandDetectionsFromPoseToRectsCalculator::Open(
CalculatorContext* cc) {
RET_CHECK(cc->Inputs().HasTag(kImageSizeTag))
<< "Image size is required to calculate rotated rect.";
@@ -72,10 +72,10 @@ constexpr char kImageSizeTag[] = "IMAGE_SIZE";
output_zero_rect_for_empty_detections_ =
options_.output_zero_rect_for_empty_detections();
return ::mediapipe::OkStatus();
return ::absl::OkStatus();
}
::mediapipe::Status
::absl::Status
HandDetectionsFromPoseToRectsCalculator ::DetectionToNormalizedRect(
const Detection& detection, const DetectionSpec& detection_spec,
NormalizedRect* rect) {
@@ -118,10 +118,10 @@ HandDetectionsFromPoseToRectsCalculator ::DetectionToNormalizedRect(
rect->set_width(box_size / image_size->first);
rect->set_height(box_size / image_size->second);
return ::mediapipe::OkStatus();
return ::absl::OkStatus();
}
::mediapipe::Status HandDetectionsFromPoseToRectsCalculator::ComputeRotation(
absl::Status HandDetectionsFromPoseToRectsCalculator::ComputeRotation(
const Detection& detection, const DetectionSpec& detection_spec,
float* rotation) {
const auto& location_data = detection.location_data();
@@ -150,7 +150,7 @@ HandDetectionsFromPoseToRectsCalculator ::DetectionToNormalizedRect(
*rotation = NormalizeRadians(
target_angle_ - std::atan2(-(y_middle - y_wrist), x_middle - x_wrist));
return ::mediapipe::OkStatus();
return ::absl::OkStatus();
}
} // namespace mediapipe
mediapipe/modules/holistic_landmark/calculators/roi_tracking_calculator.cc
+11 -11
View File
@@ -253,36 +253,36 @@ bool LandmarksRequirementsSatisfied(const NormalizedLandmarkList& landmarks,
// }
class RoiTrackingCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
static absl::Status GetContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
RoiTrackingCalculatorOptions options_;
};
REGISTER_CALCULATOR(RoiTrackingCalculator);
mediapipe::Status RoiTrackingCalculator::GetContract(CalculatorContract* cc) {
absl::Status RoiTrackingCalculator::GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kPrevLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kPrevLandmarksRectTag).Set<NormalizedRect>();
cc->Inputs().Tag(kRecropRectTag).Set<NormalizedRect>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs().Tag(kTrackingRectTag).Set<NormalizedRect>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status RoiTrackingCalculator::Open(CalculatorContext* cc) {
absl::Status RoiTrackingCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
options_ = cc->Options<RoiTrackingCalculatorOptions>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status RoiTrackingCalculator::Process(CalculatorContext* cc) {
absl::Status RoiTrackingCalculator::Process(CalculatorContext* cc) {
// If there is no current frame re-crop rect (i.e. object is not present on
// the current frame) - return empty packet.
if (cc->Inputs().Tag(kRecropRectTag).IsEmpty()) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
// If there is no previous rect, but there is current re-crop rect - return
@@ -291,7 +291,7 @@ mediapipe::Status RoiTrackingCalculator::Process(CalculatorContext* cc) {
cc->Outputs()
.Tag(kTrackingRectTag)
.AddPacket(cc->Inputs().Tag(kRecropRectTag).Value());
return mediapipe::OkStatus();
return absl::OkStatus();
}
// At this point we have both previous rect (which also means we have previous
@@ -352,7 +352,7 @@ mediapipe::Status RoiTrackingCalculator::Process(CalculatorContext* cc) {
VLOG(1) << "Lost tracking: check messages above for details";
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace mediapipe
mediapipe/modules/objectron/BUILD
+19 -8
View File
@@ -21,6 +21,17 @@ licenses(["notice"])
package(default_visibility = ["//visibility:public"])
exports_files([
"object_detection_3d_camera.tflite",
"object_detection_3d_chair.tflite",
"object_detection_3d_chair_1stage.tflite",
"object_detection_3d_cup.tflite",
"object_detection_3d_sneakers.tflite",
"object_detection_3d_sneakers_1stage.tflite",
"object_detection_oidv4_labelmap.txt",
"object_detection_ssd_mobilenetv2_oidv4_fp16.tflite",
])
mediapipe_simple_subgraph(
name = "objectron_detection_1stage_gpu",
graph = "objectron_detection_1stage_gpu.pbtxt",
@@ -98,11 +109,10 @@ mediapipe_simple_subgraph(
graph = "object_detection_oid_v4_gpu.pbtxt",
register_as = "ObjectDetectionOidV4Subgraph",
deps = [
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/tflite:tflite_converter_calculator",
"//mediapipe/calculators/tflite:tflite_inference_calculator",
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
"//mediapipe/modules/objectron/calculators:filter_detection_calculator",
@@ -114,11 +124,10 @@ mediapipe_simple_subgraph(
graph = "object_detection_oid_v4_cpu.pbtxt",
register_as = "ObjectDetectionOidV4Subgraph",
deps = [
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/tflite:tflite_converter_calculator",
"//mediapipe/calculators/tflite:tflite_inference_calculator",
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
"//mediapipe/modules/objectron/calculators:filter_detection_calculator",
@@ -139,9 +148,11 @@ mediapipe_simple_subgraph(
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:previous_loopback_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/tflite:tflite_model_calculator",
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:local_file_contents_calculator",
"//mediapipe/modules/objectron/calculators:frame_annotation_to_rect_calculator",
"//mediapipe/modules/objectron/calculators:landmarks_to_frame_annotation_calculator",
"//mediapipe/modules/objectron/calculators:lift_2d_frame_annotation_to_3d_calculator",
mediapipe/modules/objectron/calculators/BUILD
+11 -2
View File
@@ -13,6 +13,7 @@
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load("//mediapipe/framework:mediapipe_register_type.bzl", "mediapipe_register_type")
licenses(["notice"])
@@ -40,6 +41,15 @@ mediapipe_proto_library(
],
)
mediapipe_register_type(
base_name = "annotation",
include_headers = ["mediapipe/modules/objectron/calculators/annotation_data.pb.h"],
types = [
"::mediapipe::FrameAnnotation",
],
deps = [":annotation_cc_proto"],
)
mediapipe_proto_library(
name = "camera_parameters_proto",
srcs = ["camera_parameters.proto"],
@@ -153,6 +163,7 @@ cc_library(
deps = [
":annotation_cc_proto",
":belief_decoder_config_cc_proto",
":box",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
@@ -291,7 +302,6 @@ cc_library(
":decoder",
":lift_2d_frame_annotation_to_3d_calculator_cc_proto",
":tensor_util",
":tflite_tensors_to_objects_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:detection_cc_proto",
@@ -311,7 +321,6 @@ cc_library(
srcs = ["frame_annotation_to_rect_calculator.cc"],
deps = [
":annotation_cc_proto",
":box",
":frame_annotation_to_rect_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:rect_cc_proto",
mediapipe/modules/objectron/calculators/annotation_data.proto
+12
View File
@@ -56,6 +56,18 @@ message ObjectAnnotation {
// Visibiity of this annotation in a frame.
float visibility = 3;
// 3x3 row-major rotation matrix describing the orientation of the rigid
// object's frame of reference in the camera-coordinate system.
repeated float rotation = 4;
// 3x1 vector describing the translation of the rigid object's frame of
// reference in the camera-coordinate system in meters.
repeated float translation = 5;
// 3x1 vector describing the scale of the rigid object's frame of reference in
// the camera-coordinate system.
repeated float scale = 6;
}
message FrameAnnotation {
mediapipe/modules/objectron/calculators/decoder.cc
+30 -1
View File
@@ -22,6 +22,7 @@
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
#include "mediapipe/modules/objectron/calculators/box.h"
namespace mediapipe {
constexpr int Decoder::kNumOffsetmaps = 16;
@@ -203,8 +204,9 @@ absl::Status Decoder::Lift2DTo3D(
float u, v;
for (int i = 0; i < 8; ++i) {
const auto& keypoint2d = annotation.keypoints(i + 1).point_2d();
// Convert 2d point from screen coordinates to NDC coordinates([-1, 1]).
if (portrait) {
// swap x and y given that our image is in portrait orientation
// Swap x and y given that our image is in portrait orientation
u = keypoint2d.y() * 2 - 1;
v = keypoint2d.x() * 2 - 1;
} else {
@@ -237,6 +239,7 @@ absl::Status Decoder::Lift2DTo3D(
Eigen::VectorXf eigen_vec = eigen_solver.eigenvectors().col(0);
Eigen::Map<Eigen::Matrix<float, 4, 3, Eigen::RowMajor>> control_matrix(
eigen_vec.data());
// All 3d points should be in front of camera (z < 0).
if (control_matrix(0, 2) > 0) {
control_matrix = -control_matrix;
}
@@ -246,10 +249,36 @@ absl::Status Decoder::Lift2DTo3D(
// Then set the 8 vertices.
Eigen::Matrix<float, 8, 3, Eigen::RowMajor> vertices =
epnp_alpha_ * control_matrix;
std::vector<Eigen::Vector3f> vertices_vec;
vertices_vec.emplace_back(Eigen::Vector3f(
control_matrix(0, 0), control_matrix(0, 1), control_matrix(0, 2)));
for (int i = 0; i < 8; ++i) {
SetPoint3d(vertices(i, 0), vertices(i, 1), vertices(i, 2),
annotation.mutable_keypoints(i + 1)->mutable_point_3d());
vertices_vec.emplace_back(
Eigen::Vector3f(vertices(i, 0), vertices(i, 1), vertices(i, 2)));
}
// Fit a box to the vertices to get box scale, rotation, translation.
Box box("category");
box.Fit(vertices_vec);
const Eigen::Matrix<float, 3, 3, Eigen::RowMajor> rotation =
box.GetRotation();
const Eigen::Vector3f translation = box.GetTranslation();
const Eigen::Vector3f scale = box.GetScale();
// Fill box rotation.
std::vector<float> rotation_vec(rotation.data(),
rotation.data() + rotation.size());
*annotation.mutable_rotation() = {rotation_vec.begin(), rotation_vec.end()};
// Fill box translation.
std::vector<float> translation_vec(translation.data(),
translation.data() + translation.size());
*annotation.mutable_translation() = {translation_vec.begin(),
translation_vec.end()};
// Fill box scale.
std::vector<float> scale_vec(scale.data(), scale.data() + scale.size());
*annotation.mutable_scale() = {scale_vec.begin(), scale_vec.end()};
}
return absl::OkStatus();
}
mediapipe/modules/objectron/calculators/filter_detection_calculator.cc
+13 -15
View File
@@ -77,8 +77,8 @@ struct FirstGreaterComparator {
}
};
mediapipe::Status SortLabelsByDecreasingScore(const Detection& detection,
Detection* sorted_detection) {
absl::Status SortLabelsByDecreasingScore(const Detection& detection,
Detection* sorted_detection) {
RET_CHECK(sorted_detection);
RET_CHECK_EQ(detection.score_size(), detection.label_size());
if (!detection.label_id().empty()) {
@@ -110,14 +110,14 @@ mediapipe::Status SortLabelsByDecreasingScore(const Detection& detection,
sorted_detection->set_label_id(i, detection.label_id(index));
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
class FilterDetectionCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
static absl::Status GetContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
bool IsValidLabel(const std::string& label);
@@ -134,8 +134,7 @@ class FilterDetectionCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(FilterDetectionCalculator);
mediapipe::Status FilterDetectionCalculator::GetContract(
CalculatorContract* cc) {
absl::Status FilterDetectionCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -153,10 +152,10 @@ mediapipe::Status FilterDetectionCalculator::GetContract(
if (cc->InputSidePackets().HasTag(kLabelsCsvTag)) {
cc->InputSidePackets().Tag(kLabelsCsvTag).Set<std::string>();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FilterDetectionCalculator::Open(CalculatorContext* cc) {
absl::Status FilterDetectionCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
options_ = cc->Options<FilterDetectionCalculatorOptions>();
limit_labels_ = cc->InputSidePackets().HasTag(kLabelsTag) ||
@@ -187,12 +186,12 @@ mediapipe::Status FilterDetectionCalculator::Open(CalculatorContext* cc) {
limit_labels_ = false;
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FilterDetectionCalculator::Process(CalculatorContext* cc) {
absl::Status FilterDetectionCalculator::Process(CalculatorContext* cc) {
if (limit_labels_ && allowed_labels_.empty()) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
Detections detections;
if (cc->Inputs().HasTag(kDetectionsTag)) {
@@ -234,7 +233,7 @@ mediapipe::Status FilterDetectionCalculator::Process(CalculatorContext* cc) {
.Tag(kDetectionsTag)
.Add(new Detection((*outputs)[0]), cc->InputTimestamp());
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
bool FilterDetectionCalculator::IsValidLabel(const std::string& label) {
@@ -258,7 +257,6 @@ bool FilterDetectionCalculator::IsValidScore(float score) {
LOG(ERROR) << "Filter out detection with high score " << score;
return false;
}
LOG(ERROR) << "Pass detection with score " << score;
return true;
}
mediapipe/modules/objectron/calculators/frame_annotation_to_rect_calculator.cc
+18 -29
View File
@@ -21,7 +21,6 @@
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
#include "mediapipe/modules/objectron/calculators/box.h"
#include "mediapipe/modules/objectron/calculators/frame_annotation_to_rect_calculator.pb.h"
namespace mediapipe {
@@ -48,9 +47,9 @@ class FrameAnnotationToRectCalculator : public CalculatorBase {
TOP_VIEW_OFF,
};
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
static absl::Status GetContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
void AddAnnotationToRect(const ObjectAnnotation& annotation,
@@ -65,7 +64,7 @@ class FrameAnnotationToRectCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(FrameAnnotationToRectCalculator);
mediapipe::Status FrameAnnotationToRectCalculator::GetContract(
absl::Status FrameAnnotationToRectCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -77,23 +76,22 @@ mediapipe::Status FrameAnnotationToRectCalculator::GetContract(
if (cc->Outputs().HasTag(kOutputNormRectsTag)) {
cc->Outputs().Tag(kOutputNormRectsTag).Set<std::vector<NormalizedRect>>();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FrameAnnotationToRectCalculator::Open(CalculatorContext* cc) {
absl::Status FrameAnnotationToRectCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
status_ = TOP_VIEW_OFF;
const auto& options = cc->Options<FrameAnnotationToRectCalculatorOptions>();
off_threshold_ = options.off_threshold();
on_threshold_ = options.on_threshold();
RET_CHECK(off_threshold_ <= on_threshold_);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FrameAnnotationToRectCalculator::Process(
CalculatorContext* cc) {
absl::Status FrameAnnotationToRectCalculator::Process(CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputFrameAnnotationTag).IsEmpty()) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
auto output_rects = absl::make_unique<std::vector<NormalizedRect>>();
const auto& frame_annotation =
@@ -106,7 +104,7 @@ mediapipe::Status FrameAnnotationToRectCalculator::Process(
cc->Outputs()
.Tag(kOutputNormRectsTag)
.Add(output_rects.release(), cc->InputTimestamp());
return mediapipe::OkStatus();
return absl::OkStatus();
}
void FrameAnnotationToRectCalculator::AddAnnotationToRect(
@@ -133,20 +131,11 @@ void FrameAnnotationToRectCalculator::AddAnnotationToRect(
float FrameAnnotationToRectCalculator::RotationAngleFromAnnotation(
const ObjectAnnotation& annotation) {
Box box("category");
std::vector<Vector3f> vertices_3d;
std::vector<Vector2f> vertices_2d;
for (const auto& keypoint : annotation.keypoints()) {
const auto& point_3d = keypoint.point_3d();
const auto& point_2d = keypoint.point_2d();
vertices_3d.emplace_back(
Vector3f(point_3d.x(), point_3d.y(), point_3d.z()));
vertices_2d.emplace_back(Vector2f(point_2d.x(), point_2d.y()));
}
box.Fit(vertices_3d);
Vector3f scale = box.GetScale();
Matrix3fRM box_rotation = box.GetRotation();
Vector3f box_translation = box.GetTranslation();
// Get box rotation and translation from annotation.
const auto box_rotation =
Eigen::Map<const Matrix3fRM>(annotation.rotation().data());
const auto box_translation =
Eigen::Map<const Vector3f>(annotation.translation().data());
// Rotation angle to use when top-view is on(top-view on),
// Which will make z-axis upright after the rotation.
@@ -180,9 +169,9 @@ float FrameAnnotationToRectCalculator::RotationAngleFromPose(
const Vector3f& vec) {
auto p1 = rotation * vec + translation;
auto p2 = -rotation * vec + translation;
const float dy = p2[2] * p1[1] - p1[2] * p2[1];
const float dx = p2[2] * p1[0] - p1[2] * p2[0];
return std::atan2(-dy, dx);
const float dy = p2[2] * p2[1] - p1[2] * p1[1];
const float dx = p2[2] * p2[0] - p1[2] * p1[0];
return M_PI / 2 - std::atan2(dy, dx);
}
} // namespace mediapipe
mediapipe/modules/objectron/calculators/frame_annotation_to_timed_box_list_calculator.cc
+12 -12
View File
@@ -47,15 +47,15 @@ namespace mediapipe {
// }
class FrameAnnotationToTimedBoxListCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc);
static absl::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
};
REGISTER_CALCULATOR(FrameAnnotationToTimedBoxListCalculator);
mediapipe::Status FrameAnnotationToTimedBoxListCalculator::GetContract(
absl::Status FrameAnnotationToTimedBoxListCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -67,15 +67,15 @@ mediapipe::Status FrameAnnotationToTimedBoxListCalculator::GetContract(
if (cc->Outputs().HasTag(kOutputStreamTag)) {
cc->Outputs().Tag(kOutputStreamTag).Set<TimedBoxProtoList>();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Open(
absl::Status FrameAnnotationToTimedBoxListCalculator::Open(
CalculatorContext* cc) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Process(
absl::Status FrameAnnotationToTimedBoxListCalculator::Process(
CalculatorContext* cc) {
if (cc->Inputs().HasTag(kInputStreamTag) &&
!cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
@@ -104,12 +104,12 @@ mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Process(
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Close(
absl::Status FrameAnnotationToTimedBoxListCalculator::Close(
CalculatorContext* cc) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace mediapipe
mediapipe/modules/objectron/calculators/frame_annotation_tracker_calculator.cc
+12 -15
View File
@@ -52,18 +52,18 @@ namespace mediapipe {
// }
class FrameAnnotationTrackerCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc);
static absl::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
private:
std::unique_ptr<FrameAnnotationTracker> frame_annotation_tracker_;
};
REGISTER_CALCULATOR(FrameAnnotationTrackerCalculator);
mediapipe::Status FrameAnnotationTrackerCalculator::GetContract(
absl::Status FrameAnnotationTrackerCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -80,19 +80,17 @@ mediapipe::Status FrameAnnotationTrackerCalculator::GetContract(
if (cc->Outputs().HasTag(kOutputCancelObjectIdTag)) {
cc->Outputs().Tag(kOutputCancelObjectIdTag).Set<int>();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FrameAnnotationTrackerCalculator::Open(
CalculatorContext* cc) {
absl::Status FrameAnnotationTrackerCalculator::Open(CalculatorContext* cc) {
const auto& options = cc->Options<FrameAnnotationTrackerCalculatorOptions>();
frame_annotation_tracker_ = absl::make_unique<FrameAnnotationTracker>(
options.iou_threshold(), options.img_width(), options.img_height());
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FrameAnnotationTrackerCalculator::Process(
CalculatorContext* cc) {
absl::Status FrameAnnotationTrackerCalculator::Process(CalculatorContext* cc) {
if (cc->Inputs().HasTag(kInputFrameAnnotationTag) &&
!cc->Inputs().Tag(kInputFrameAnnotationTag).IsEmpty()) {
frame_annotation_tracker_->AddDetectionResult(
@@ -126,12 +124,11 @@ mediapipe::Status FrameAnnotationTrackerCalculator::Process(
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FrameAnnotationTrackerCalculator::Close(
CalculatorContext* cc) {
return mediapipe::OkStatus();
absl::Status FrameAnnotationTrackerCalculator::Close(CalculatorContext* cc) {
return absl::OkStatus();
}
} // namespace mediapipe
mediapipe/modules/objectron/calculators/landmarks_to_frame_annotation_calculator.cc
+9 -10
View File
@@ -31,9 +31,9 @@ constexpr char kOutputFrameAnnotationTag[] = "FRAME_ANNOTATION";
// A calculator that converts NormalizedLandmarkList to FrameAnnotation proto.
class LandmarksToFrameAnnotationCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
static absl::Status GetContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
void AddLandmarksToFrameAnnotation(const NormalizedLandmarkList& landmarks,
@@ -41,7 +41,7 @@ class LandmarksToFrameAnnotationCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(LandmarksToFrameAnnotationCalculator);
mediapipe::Status LandmarksToFrameAnnotationCalculator::GetContract(
absl::Status LandmarksToFrameAnnotationCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -57,16 +57,15 @@ mediapipe::Status LandmarksToFrameAnnotationCalculator::GetContract(
if (cc->Outputs().HasTag(kOutputFrameAnnotationTag)) {
cc->Outputs().Tag(kOutputFrameAnnotationTag).Set<FrameAnnotation>();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status LandmarksToFrameAnnotationCalculator::Open(
CalculatorContext* cc) {
absl::Status LandmarksToFrameAnnotationCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status LandmarksToFrameAnnotationCalculator::Process(
absl::Status LandmarksToFrameAnnotationCalculator::Process(
CalculatorContext* cc) {
auto frame_annotation = absl::make_unique<FrameAnnotation>();
@@ -96,7 +95,7 @@ mediapipe::Status LandmarksToFrameAnnotationCalculator::Process(
.Tag(kOutputFrameAnnotationTag)
.Add(frame_annotation.release(), cc->InputTimestamp());
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
void LandmarksToFrameAnnotationCalculator::AddLandmarksToFrameAnnotation(
mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.cc
+30 -29
View File
@@ -55,16 +55,16 @@ namespace mediapipe {
// }
class Lift2DFrameAnnotationTo3DCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc);
static absl::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
private:
mediapipe::Status ProcessCPU(CalculatorContext* cc,
FrameAnnotation* output_objects);
mediapipe::Status LoadOptions(CalculatorContext* cc);
absl::Status ProcessCPU(CalculatorContext* cc,
FrameAnnotation* output_objects);
absl::Status LoadOptions(CalculatorContext* cc);
// Increment and assign object ID for each detected object.
// In a single MediaPipe session, the IDs are unique.
@@ -78,37 +78,39 @@ class Lift2DFrameAnnotationTo3DCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(Lift2DFrameAnnotationTo3DCalculator);
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::GetContract(
absl::Status Lift2DFrameAnnotationTo3DCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag(kInputStreamTag));
RET_CHECK(cc->Outputs().HasTag(kOutputStreamTag));
cc->Inputs().Tag(kInputStreamTag).Set<FrameAnnotation>();
cc->Outputs().Tag(kOutputStreamTag).Set<FrameAnnotation>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Open(
CalculatorContext* cc) {
absl::Status Lift2DFrameAnnotationTo3DCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
MP_RETURN_IF_ERROR(LoadOptions(cc));
// Load camera intrinsic matrix.
const float fx = options_.normalized_focal_x();
const float fy = options_.normalized_focal_y();
const float px = options_.normalized_principal_point_x();
const float py = options_.normalized_principal_point_y();
// clang-format off
projection_matrix_ <<
1.5731, 0, 0, 0,
0, 2.0975, 0, 0,
0, 0, -1.0002, -0.2,
0, 0, -1, 0;
projection_matrix_ << fx, 0., px, 0.,
0., fy, py, 0.,
0., 0., -1., 0.,
0., 0., -1., 0.;
// clang-format on
decoder_ = absl::make_unique<Decoder>(
BeliefDecoderConfig(options_.decoder_config()));
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Process(
absl::Status Lift2DFrameAnnotationTo3DCalculator::Process(
CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
auto output_objects = absl::make_unique<FrameAnnotation>();
@@ -122,17 +124,17 @@ mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Process(
.Add(output_objects.release(), cc->InputTimestamp());
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::ProcessCPU(
absl::Status Lift2DFrameAnnotationTo3DCalculator::ProcessCPU(
CalculatorContext* cc, FrameAnnotation* output_objects) {
const auto& input_frame_annotations =
cc->Inputs().Tag(kInputStreamTag).Get<FrameAnnotation>();
// Copy the input frame annotation to the output
*output_objects = input_frame_annotations;
auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ true,
auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ false,
output_objects);
if (!status.ok()) {
LOG(ERROR) << status;
@@ -141,20 +143,19 @@ mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::ProcessCPU(
AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(),
output_objects);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Close(
CalculatorContext* cc) {
return mediapipe::OkStatus();
absl::Status Lift2DFrameAnnotationTo3DCalculator::Close(CalculatorContext* cc) {
return absl::OkStatus();
}
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::LoadOptions(
absl::Status Lift2DFrameAnnotationTo3DCalculator::LoadOptions(
CalculatorContext* cc) {
// Get calculator options specified in the graph.
options_ = cc->Options<Lift2DFrameAnnotationTo3DCalculatorOptions>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
void Lift2DFrameAnnotationTo3DCalculator::AssignObjectIdAndTimestamp(
mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.proto
+12
View File
@@ -27,4 +27,16 @@ message Lift2DFrameAnnotationTo3DCalculatorOptions {
}
optional BeliefDecoderConfig decoder_config = 1;
// Camera focal length along x, normalized by width/2.
optional float normalized_focal_x = 2 [default = 1.0];
// Camera focal length along y, normalized by height/2.
optional float normalized_focal_y = 3 [default = 1.0];
// Camera principle point x, normalized by width/2, origin is image center.
optional float normalized_principal_point_x = 4 [default = 0.0];
// Camera principle point y, normalized by height/2, origin is image center.
optional float normalized_principal_point_y = 5 [default = 0.0];
}
mediapipe/modules/objectron/calculators/tensors_to_objects_calculator.cc
+20 -22
View File
@@ -58,16 +58,16 @@ namespace mediapipe {
// }
class TensorsToObjectsCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc);
static absl::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
private:
mediapipe::Status ProcessCPU(CalculatorContext* cc,
FrameAnnotation* output_objects);
mediapipe::Status LoadOptions(CalculatorContext* cc);
absl::Status ProcessCPU(CalculatorContext* cc,
FrameAnnotation* output_objects);
absl::Status LoadOptions(CalculatorContext* cc);
// Takes point_3d in FrameAnnotation, projects to 2D, and overwrite the
// point_2d field with the projection.
void Project3DTo2D(bool portrait, FrameAnnotation* annotation) const;
@@ -87,8 +87,7 @@ class TensorsToObjectsCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(TensorsToObjectsCalculator);
mediapipe::Status TensorsToObjectsCalculator::GetContract(
CalculatorContract* cc) {
absl::Status TensorsToObjectsCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -99,10 +98,10 @@ mediapipe::Status TensorsToObjectsCalculator::GetContract(
if (cc->Outputs().HasTag(kOutputStreamTag)) {
cc->Outputs().Tag(kOutputStreamTag).Set<FrameAnnotation>();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status TensorsToObjectsCalculator::Open(CalculatorContext* cc) {
absl::Status TensorsToObjectsCalculator::Open(CalculatorContext* cc) {
MP_RETURN_IF_ERROR(LoadOptions(cc));
// clang-format off
projection_matrix_ <<
@@ -114,12 +113,12 @@ mediapipe::Status TensorsToObjectsCalculator::Open(CalculatorContext* cc) {
decoder_ = absl::make_unique<Decoder>(
BeliefDecoderConfig(options_.decoder_config()));
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status TensorsToObjectsCalculator::Process(CalculatorContext* cc) {
absl::Status TensorsToObjectsCalculator::Process(CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
auto output_objects = absl::make_unique<FrameAnnotation>();
@@ -133,10 +132,10 @@ mediapipe::Status TensorsToObjectsCalculator::Process(CalculatorContext* cc) {
.Add(output_objects.release(), cc->InputTimestamp());
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status TensorsToObjectsCalculator::ProcessCPU(
absl::Status TensorsToObjectsCalculator::ProcessCPU(
CalculatorContext* cc, FrameAnnotation* output_objects) {
const auto& input_tensors =
cc->Inputs().Tag(kInputStreamTag).Get<std::vector<mediapipe::Tensor>>();
@@ -156,15 +155,14 @@ mediapipe::Status TensorsToObjectsCalculator::ProcessCPU(
AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(),
output_objects);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status TensorsToObjectsCalculator::Close(CalculatorContext* cc) {
return mediapipe::OkStatus();
absl::Status TensorsToObjectsCalculator::Close(CalculatorContext* cc) {
return absl::OkStatus();
}
mediapipe::Status TensorsToObjectsCalculator::LoadOptions(
CalculatorContext* cc) {
absl::Status TensorsToObjectsCalculator::LoadOptions(CalculatorContext* cc) {
// Get calculator options specified in the graph.
options_ = cc->Options<::mediapipe::TensorsToObjectsCalculatorOptions>();
@@ -174,7 +172,7 @@ mediapipe::Status TensorsToObjectsCalculator::LoadOptions(
// Currently only support 2D when num_values_per_keypoint equals to 2.
CHECK_EQ(options_.num_values_per_keypoint(), 2);
return mediapipe::OkStatus();
return absl::OkStatus();
}
void TensorsToObjectsCalculator::Project3DTo2D(
mediapipe/modules/objectron/calculators/tflite_tensors_to_objects_calculator.cc
+29 -28
View File
@@ -59,16 +59,16 @@ namespace mediapipe {
// }
class TfLiteTensorsToObjectsCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc);
static absl::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
private:
mediapipe::Status ProcessCPU(CalculatorContext* cc,
FrameAnnotation* output_objects);
mediapipe::Status LoadOptions(CalculatorContext* cc);
absl::Status ProcessCPU(CalculatorContext* cc,
FrameAnnotation* output_objects);
absl::Status LoadOptions(CalculatorContext* cc);
// Takes point_3d in FrameAnnotation, projects to 2D, and overwrite the
// point_2d field with the projection.
void Project3DTo2D(bool portrait, FrameAnnotation* annotation) const;
@@ -88,7 +88,7 @@ class TfLiteTensorsToObjectsCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(TfLiteTensorsToObjectsCalculator);
mediapipe::Status TfLiteTensorsToObjectsCalculator::GetContract(
absl::Status TfLiteTensorsToObjectsCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -100,29 +100,31 @@ mediapipe::Status TfLiteTensorsToObjectsCalculator::GetContract(
if (cc->Outputs().HasTag(kOutputStreamTag)) {
cc->Outputs().Tag(kOutputStreamTag).Set<FrameAnnotation>();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status TfLiteTensorsToObjectsCalculator::Open(
CalculatorContext* cc) {
absl::Status TfLiteTensorsToObjectsCalculator::Open(CalculatorContext* cc) {
MP_RETURN_IF_ERROR(LoadOptions(cc));
// Load camera intrinsic matrix.
const float fx = options_.normalized_focal_x();
const float fy = options_.normalized_focal_y();
const float px = options_.normalized_principal_point_x();
const float py = options_.normalized_principal_point_y();
// clang-format off
projection_matrix_ <<
1.5731, 0, 0, 0,
0, 2.0975, 0, 0,
0, 0, -1.0002, -0.2,
0, 0, -1, 0;
projection_matrix_ << fx, 0., px, 0.,
0., fy, py, 0.,
0., 0., -1., 0.,
0., 0., -1., 0.;
// clang-format on
decoder_ = absl::make_unique<Decoder>(
BeliefDecoderConfig(options_.decoder_config()));
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status TfLiteTensorsToObjectsCalculator::Process(
CalculatorContext* cc) {
absl::Status TfLiteTensorsToObjectsCalculator::Process(CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
auto output_objects = absl::make_unique<FrameAnnotation>();
@@ -136,10 +138,10 @@ mediapipe::Status TfLiteTensorsToObjectsCalculator::Process(
.Add(output_objects.release(), cc->InputTimestamp());
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status TfLiteTensorsToObjectsCalculator::ProcessCPU(
absl::Status TfLiteTensorsToObjectsCalculator::ProcessCPU(
CalculatorContext* cc, FrameAnnotation* output_objects) {
const auto& input_tensors =
cc->Inputs().Tag(kInputStreamTag).Get<std::vector<TfLiteTensor>>();
@@ -159,15 +161,14 @@ mediapipe::Status TfLiteTensorsToObjectsCalculator::ProcessCPU(
AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(),
output_objects);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status TfLiteTensorsToObjectsCalculator::Close(
CalculatorContext* cc) {
return mediapipe::OkStatus();
absl::Status TfLiteTensorsToObjectsCalculator::Close(CalculatorContext* cc) {
return absl::OkStatus();
}
mediapipe::Status TfLiteTensorsToObjectsCalculator::LoadOptions(
absl::Status TfLiteTensorsToObjectsCalculator::LoadOptions(
CalculatorContext* cc) {
// Get calculator options specified in the graph.
options_ =
@@ -179,7 +180,7 @@ mediapipe::Status TfLiteTensorsToObjectsCalculator::LoadOptions(
// Currently only support 2D when num_values_per_keypoint equals to 2.
CHECK_EQ(options_.num_values_per_keypoint(), 2);
return mediapipe::OkStatus();
return absl::OkStatus();
}
void TfLiteTensorsToObjectsCalculator::Project3DTo2D(
mediapipe/modules/objectron/calculators/tflite_tensors_to_objects_calculator.proto
+12
View File
@@ -36,4 +36,16 @@ message TfLiteTensorsToObjectsCalculatorOptions {
optional int32 num_values_per_keypoint = 3 [default = 2];
optional BeliefDecoderConfig decoder_config = 4;
// Camera focal length along x, normalized by width/2.
optional float normalized_focal_x = 5 [default = 1.0];
// Camera focal length along y, normalized by height/2.
optional float normalized_focal_y = 6 [default = 1.0];
// Camera principle point x, normalized by width/2, origin is image center.
optional float normalized_principal_point_x = 7 [default = 0.0];
// Camera principle point y, normalized by height/2, origin is image center.
optional float normalized_principal_point_y = 8 [default = 0.0];
}
mediapipe/modules/objectron/object_detection_3d_camera.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/objectron/object_detection_3d_chair.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/objectron/object_detection_3d_cup.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/objectron/object_detection_3d_sneakers.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/objectron/object_detection_oid_v4_cpu.pbtxt
+23 -27
View File
@@ -6,41 +6,37 @@ input_stream: "IMAGE:input_video"
input_side_packet: "LABELS_CSV:allowed_labels"
output_stream: "DETECTIONS:detections"
# Transforms the input image on CPU to a 300x300 image. To scale the image, by
# default it uses the STRETCH scale mode that maps the entire input image to the
# entire transformed image. As a result, image aspect ratio may be changed and
# objects in the image may be deformed (stretched or squeezed), but the object
# detection model used in this graph is agnostic to that deformation.
node: {
calculator: "ImageTransformationCalculator"
# Crops, resizes, and converts the input video into tensor.
# Preserves aspect ratio of the images.
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:input_video"
output_stream: "IMAGE:transformed_input_video"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext] {
output_width: 300
output_height: 300
output_stream: "TENSORS:image_tensor"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 300
output_tensor_height: 300
keep_aspect_ratio: false
output_tensor_float_range {
min: -1.0
max: 1.0
}
}
}
}
# Converts the transformed input image on CPU into an image tensor stored as a
# TfLiteTensor.
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE:transformed_input_video"
output_stream: "TENSORS:image_tensor"
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "TfLiteInferenceCalculator"
calculator: "InferenceCalculator"
input_stream: "TENSORS:image_tensor"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "mediapipe/models/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/objectron/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
delegate { xnnpack {} }
}
}
}
@@ -79,13 +75,13 @@ node {
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TfLiteTensorsToDetectionsCalculator"
calculator: "TensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:all_detections"
options: {
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
num_classes: 195
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 24
num_boxes: 1917
num_coords: 4
ignore_classes: 0
@@ -108,7 +104,7 @@ node {
output_stream: "labeled_detections"
options: {
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
label_map_path: "mediapipe/models/object_detection_oidv4_labelmap.pbtxt"
label_map_path: "mediapipe/modules/objectron/object_detection_oidv4_labelmap.txt"
}
}
}
mediapipe/modules/objectron/object_detection_oid_v4_gpu.pbtxt
+27 -29
View File
@@ -6,41 +6,39 @@ input_stream: "IMAGE_GPU:input_video"
input_side_packet: "LABELS_CSV:allowed_labels"
output_stream: "DETECTIONS:detections"
# Transforms the input image on GPU to a 300x300 image. To scale the image, by
# default it uses the STRETCH scale mode that maps the entire input image to the
# entire transformed image. As a result, image aspect ratio may be changed and
# objects in the image may be deformed (stretched or squeezed), but the object
# detection model used in this graph is agnostic to that deformation.
node: {
calculator: "ImageTransformationCalculator"
# Crops, resizes, and converts the input video into tensor.
# Preserves aspect ratio of the images.
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE_GPU:input_video"
output_stream: "IMAGE_GPU:transformed_input_video"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext] {
output_width: 300
output_height: 300
output_stream: "TENSORS:image_tensor"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 300
output_tensor_height: 300
keep_aspect_ratio: false
output_tensor_float_range {
min: -1.0
max: 1.0
}
gpu_origin: TOP_LEFT
}
}
}
# Converts the transformed input image on GPU into an image tensor stored as a
# TfLiteTensor.
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE_GPU:transformed_input_video"
output_stream: "TENSORS_GPU:image_tensor"
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS_GPU:image_tensor"
output_stream: "TENSORS_GPU:detection_tensors"
calculator: "InferenceCalculator"
input_stream: "TENSORS:image_tensor"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/objectron/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
delegate { gpu {} }
}
}
}
@@ -79,13 +77,13 @@ node {
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TfLiteTensorsToDetectionsCalculator"
input_stream: "TENSORS_GPU:detection_tensors"
calculator: "TensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:all_detections"
options: {
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
num_classes: 195
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 24
num_boxes: 1917
num_coords: 4
ignore_classes: 0
@@ -108,7 +106,7 @@ node {
output_stream: "labeled_detections"
options: {
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
label_map_path: "object_detection_oidv4_labelmap.pbtxt"
label_map_path: "object_detection_oidv4_labelmap.txt"
}
}
}
mediapipe/modules/objectron/object_detection_oidv4_labelmap.txt
+24
View File
@@ -0,0 +1,24 @@
???
Bicycle
Boot
Laptop
Person
Chair
Cattle
Desk
Cat
Computer mouse
Computer monitor
Box
Mug
Coffee cup
Stationary bicycle
Table
Bottle
High heels
Vehicle
Footwear
Dog
Book
Camera
Car
mediapipe/modules/objectron/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/objectron/objectron_cpu.pbtxt
+42 -15
View File
@@ -1,7 +1,10 @@
# MediaPipe Objectron on CPU that produces 3D bounding boxes for objects.
input_stream: "IMAGE:input_video"
# TfLite model for 3D bounding box landmark prediction
input_side_packet: "MODEL:box_landmark_model"
type: "ObjectronCpuSubgraph"
# Input/Output streams and input side packets.
input_stream: "IMAGE:image"
# Path to TfLite model for 3D bounding box landmark prediction
input_side_packet: "MODEL_PATH:box_landmark_model_path"
# Allowed category labels, e.g. Footwear, Coffee cup, Mug, Chair, Camera
input_side_packet: "LABELS_CSV:allowed_labels"
# Max number of objects to detect/track. (int)
@@ -23,11 +26,28 @@ input_side_packet: "MAX_NUM_OBJECTS:max_num_objects"
# \ + \ + \
# \+ \+
# 2 + + + + + + + + 6
#
# Collection of detected 3D objects, represented as a FrameAnnotation.
output_stream: "FRAME_ANNOTATION:detected_objects"
# Collection of box landmarks. (NormalizedLandmarkList)
output_stream: "MULTI_LANDMARKS:multi_box_landmarks"
# Crop rectangles derived from bounding box landmarks.
output_stream: "NORM_RECTS:multi_box_rects"
# Loads the file in the specified path into a blob.
node {
calculator: "LocalFileContentsCalculator"
input_side_packet: "FILE_PATH:0:box_landmark_model_path"
output_side_packet: "CONTENTS:0:box_landmark_model_blob"
}
# Converts the input blob into a TF Lite model.
node {
calculator: "TfLiteModelCalculator"
input_side_packet: "MODEL_BLOB:box_landmark_model_blob"
output_side_packet: "MODEL:box_landmark_model"
}
# Defines whether landmarks from the previous video frame should be used to help
# predict landmarks on the current video frame.
node {
@@ -62,9 +82,9 @@ node {
# to trigger a new round of box detection in ObjectDetectionOidV4Subgraph.
node {
calculator: "GateCalculator"
input_stream: "input_video"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_objects"
output_stream: "detection_input_video"
output_stream: "detection_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
@@ -76,7 +96,7 @@ node {
# Subgraph that performs 2D object detection.
node {
calculator: "ObjectDetectionOidV4Subgraph"
input_stream: "IMAGE:detection_input_video"
input_stream: "IMAGE:detection_image"
input_side_packet: "LABELS_CSV:allowed_labels"
output_stream: "DETECTIONS:raw_detections"
}
@@ -93,7 +113,7 @@ node {
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:input_video"
input_stream: "IMAGE:image"
output_stream: "SIZE:image_size"
}
@@ -135,16 +155,16 @@ node {
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:multi_box_rects"
input_stream: "CLONE:input_video"
input_stream: "CLONE:image"
output_stream: "ITEM:single_box_rect"
output_stream: "CLONE:landmarks_input_video"
output_stream: "CLONE:landmarks_image"
output_stream: "BATCH_END:box_rects_timestamp"
}
# Subgraph that localizes box landmarks.
node {
calculator: "BoxLandmarkSubgraph"
input_stream: "IMAGE:landmarks_input_video"
input_stream: "IMAGE:landmarks_image"
input_side_packet: "MODEL:box_landmark_model"
input_stream: "NORM_RECT:single_box_rect"
output_stream: "NORM_LANDMARKS:single_box_landmarks"
@@ -169,15 +189,22 @@ node {
# Lift the 2D landmarks to 3D using EPnP algorithm.
node {
name: "Lift2DFrameAnnotationTo3DCalculator"
calculator: "Lift2DFrameAnnotationTo3DCalculator"
input_stream: "FRAME_ANNOTATION:box_annotations"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_objects"
output_stream: "LIFTED_FRAME_ANNOTATION:detected_objects"
options: {
[mediapipe.Lift2DFrameAnnotationTo3DCalculatorOptions.ext] {
normalized_focal_x: 1.0
normalized_focal_y: 1.0
}
}
}
# Get rotated rectangle from lifted box.
# Get rotated rectangle from detected box.
node {
calculator: "FrameAnnotationToRectCalculator"
input_stream: "FRAME_ANNOTATION:lifted_objects"
input_stream: "FRAME_ANNOTATION:detected_objects"
output_stream: "NORM_RECTS:box_rects_from_landmarks"
}
@@ -189,7 +216,7 @@ node {
# feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:input_video"
input_stream: "MAIN:image"
input_stream: "LOOP:box_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
mediapipe/modules/objectron/objectron_detection_1stage_gpu.pbtxt
+2
View File
@@ -76,6 +76,8 @@ node {
voting_allowance: 1
voting_threshold: 0.2
}
normalized_focal_x: 2.0975
normalized_focal_y: 1.5731
}
}
}
mediapipe/modules/objectron/objectron_gpu.pbtxt
+21 -14
View File
@@ -1,15 +1,16 @@
# MediaPipe Objectron on GPU that produces 3D bounding boxes for objects.
type: "ObjectronGpuSubgraph"
# Input/Output streams and input side packets.
# Note that the input video is assumed to have aspect ratio 3:4 (width:height).
input_stream: "IMAGE_GPU:input_video"
# Note that the input image is assumed to have aspect ratio 3:4 (width:height).
input_stream: "IMAGE_GPU:image"
# Allowed category labels, e.g. Footwear, Coffee cup, Mug, Chair, Camera
input_side_packet: "LABELS_CSV:allowed_labels"
# Max number of objects to detect/track. (int)
input_side_packet: "MAX_NUM_OBJECTS:max_num_objects"
# Collection of detected 3D objects, represented as a FrameAnnotation.
output_stream: "FRAME_ANNOTATION:lifted_objects"
output_stream: "FRAME_ANNOTATION:detected_objects"
# Defines whether landmarks from the previous video frame should be used to help
# predict landmarks on the current video frame.
@@ -45,9 +46,9 @@ node {
# to trigger a new round of box detection in ObjectDetectionOidV4Subgraph.
node {
calculator: "GateCalculator"
input_stream: "input_video"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_objects"
output_stream: "detection_input_video"
output_stream: "detection_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
@@ -59,7 +60,7 @@ node {
# Subgraph that performs 2D object detection.
node {
calculator: "ObjectDetectionOidV4Subgraph"
input_stream: "IMAGE_GPU:detection_input_video"
input_stream: "IMAGE_GPU:detection_image"
input_side_packet: "LABELS_CSV:allowed_labels"
output_stream: "DETECTIONS:raw_detections"
}
@@ -76,7 +77,7 @@ node {
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_video"
input_stream: "IMAGE_GPU:image"
output_stream: "SIZE:image_size"
}
@@ -118,16 +119,16 @@ node {
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:box_rects"
input_stream: "CLONE:input_video"
input_stream: "CLONE:image"
output_stream: "ITEM:single_box_rect"
output_stream: "CLONE:landmarks_input_video"
output_stream: "CLONE:landmarks_image"
output_stream: "BATCH_END:box_rects_timestamp"
}
# Subgraph that localizes box landmarks.
node {
calculator: "BoxLandmarkSubgraph"
input_stream: "IMAGE:landmarks_input_video"
input_stream: "IMAGE:landmarks_image"
input_stream: "NORM_RECT:single_box_rect"
output_stream: "NORM_LANDMARKS:single_box_landmarks"
}
@@ -153,13 +154,19 @@ node {
node {
calculator: "Lift2DFrameAnnotationTo3DCalculator"
input_stream: "FRAME_ANNOTATION:box_annotations"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_objects"
output_stream: "LIFTED_FRAME_ANNOTATION:detected_objects"
options: {
[mediapipe.Lift2DFrameAnnotationTo3DCalculatorOptions.ext] {
normalized_focal_x: 2.0975
normalized_focal_y: 1.5731
}
}
}
# Get rotated rectangle from lifted box.
# Get rotated rectangle from detected box.
node {
calculator: "FrameAnnotationToRectCalculator"
input_stream: "FRAME_ANNOTATION:lifted_objects"
input_stream: "FRAME_ANNOTATION:detected_objects"
output_stream: "NORM_RECTS:box_rects_from_landmarks"
}
@@ -171,7 +178,7 @@ node {
# feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:input_video"
input_stream: "MAIN:image"
input_stream: "LOOP:box_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
mediapipe/modules/objectron/objectron_tracking_1stage_gpu.pbtxt
+6
View File
@@ -167,4 +167,10 @@ node {
calculator: "Lift2DFrameAnnotationTo3DCalculator"
input_stream: "FRAME_ANNOTATION:tracked_objects"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
options: {
[mediapipe.Lift2DFrameAnnotationTo3DCalculatorOptions.ext] {
normalized_focal_x: 2.0975
normalized_focal_y: 1.5731
}
}
}
mediapipe/modules/palm_detection/README → mediapipe/modules/palm_detection/README.md
View File
mediapipe/modules/pose_detection/pose_detection_cpu.pbtxt
+4
View File
@@ -54,6 +54,10 @@ node: {
max: 1.0
}
border_mode: BORDER_ZERO
# If this calculator truly operates in the CPU, then gpu_origin is
# ignored, but if some build switch insists on GPU inference, then we will
# still need to set this.
gpu_origin: TOP_LEFT
}
}
}
mediapipe/modules/pose_landmark/pose_landmark_full_body.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/pose_landmark/pose_landmark_model_loader.pbtxt
+1 -1
View File
@@ -48,7 +48,7 @@ node {
output_side_packet: "CONTENTS:model_blob"
options: {
[mediapipe.LocalFileContentsCalculatorOptions.ext]: {
read_as_binary: true
text_mode: false
}
}
}