talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 6e5aa035cd1f6a9333962df5d3ab97a05bd5744e
This commit is contained in:
MediaPipe Team
2022-06-28 12:11:05 +00:00
committed by Sebastian Schmidt
parent 4a20e9909d
commit c688862570
144 changed files with 5772 additions and 2118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/modules/face_detection/BUILD
+60 -35
View File
@@ -16,6 +16,8 @@ load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_simple_subgraph",
)
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
licenses(["notice"])
@@ -26,7 +28,7 @@ mediapipe_simple_subgraph(
graph = "face_detection_short_range_by_roi_cpu.pbtxt",
register_as = "FaceDetectionShortRangeByRoiCpu",
deps = [
":face_detection_short_range_common",
":face_detection_short_range",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
@@ -38,7 +40,7 @@ mediapipe_simple_subgraph(
graph = "face_detection_short_range_by_roi_gpu.pbtxt",
register_as = "FaceDetectionShortRangeByRoiGpu",
deps = [
":face_detection_short_range_common",
":face_detection_short_range",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
@@ -50,10 +52,7 @@ mediapipe_simple_subgraph(
graph = "face_detection_short_range_cpu.pbtxt",
register_as = "FaceDetectionShortRangeCpu",
deps = [
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
":face_detection_short_range",
],
)
@@ -62,22 +61,66 @@ mediapipe_simple_subgraph(
graph = "face_detection_short_range_gpu.pbtxt",
register_as = "FaceDetectionShortRangeGpu",
deps = [
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
":face_detection_short_range",
],
)
mediapipe_simple_subgraph(
name = "face_detection_short_range_common",
graph = "face_detection_short_range_common.pbtxt",
register_as = "FaceDetectionShortRangeCommon",
name = "face_detection_short_range",
graph = "face_detection_short_range.pbtxt",
register_as = "FaceDetectionShortRange",
deps = [
":face_detection",
],
)
mediapipe_simple_subgraph(
name = "face_detection_full_range",
graph = "face_detection_full_range.pbtxt",
register_as = "FaceDetectionFullRange",
deps = [
":face_detection",
],
)
mediapipe_simple_subgraph(
name = "face_detection_without_roi",
graph = "face_detection_without_roi.pbtxt",
register_as = "FaceDetectionWithoutRoi",
deps = [
":face_detection",
],
)
mediapipe_simple_subgraph(
name = "face_detection",
graph = "face_detection.pbtxt",
register_as = "FaceDetection",
deps = [
":face_detection_cc_proto",
":face_detection_options_lib",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/util:detection_projection_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_proto_library(
name = "face_detection_proto",
srcs = ["face_detection.proto"],
deps = [
"//mediapipe/calculators/core:gate_calculator_proto",
"//mediapipe/calculators/tensor:image_to_tensor_calculator_proto",
"//mediapipe/calculators/tensor:inference_calculator_proto",
"//mediapipe/calculators/tensor:tensors_to_detections_calculator_proto",
"//mediapipe/calculators/tflite:ssd_anchors_calculator_proto",
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/gpu:gpu_origin_proto",
],
)
@@ -86,10 +129,7 @@ mediapipe_simple_subgraph(
graph = "face_detection_full_range_cpu.pbtxt",
register_as = "FaceDetectionFullRangeCpu",
deps = [
":face_detection_full_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
":face_detection_full_range",
],
)
@@ -98,22 +138,7 @@ mediapipe_simple_subgraph(
graph = "face_detection_full_range_gpu.pbtxt",
register_as = "FaceDetectionFullRangeGpu",
deps = [
":face_detection_full_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_full_range_common",
graph = "face_detection_full_range_common.pbtxt",
register_as = "FaceDetectionFullRangeCommon",
deps = [
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/util:detection_projection_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
":face_detection_full_range",
],
)
@@ -122,7 +147,7 @@ mediapipe_simple_subgraph(
graph = "face_detection_short_range_image.pbtxt",
register_as = "FaceDetectionShortRangeImage",
deps = [
":face_detection_short_range_common",
":face_detection_short_range",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
@@ -134,7 +159,7 @@ mediapipe_simple_subgraph(
graph = "face_detection_full_range_image.pbtxt",
register_as = "FaceDetectionFullRangeImage",
deps = [
":face_detection_full_range_common",
":face_detection_full_range",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
mediapipe/modules/face_detection/face_detection.pbtxt
+164
View File
@@ -0,0 +1,164 @@
# MediaPipe graph to detect faces.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFrontCpu"
# input_stream: "IMAGE:image"
# input_stream: "ROI:roi"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetection"
# The input image, either ImageFrame, GpuBuffer, or (multi-backend) Image.
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where faces should be
# detected. (NormalizedRect)
input_stream: "ROI:roi"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
# Converts the input CPU or GPU image to the multi-backend image type (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
input_stream: "NORM_RECT:roi"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
}
}
option_value: "gpu_origin:options/gpu_origin"
option_value: "output_tensor_width:options/tensor_width"
option_value: "output_tensor_height:options/tensor_height"
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {}
}
option_value: "delegate:options/delegate"
option_value: "model_path:options/model_path"
}
# Detection tensors. (std::vector<Tensor>)
#input_stream: "TENSORS:detection_tensors"
# A 4x4 row-major-order matrix that maps a point represented in the detection
# tensors to a desired coordinate system, e.g., in the original input image
# before scaling/cropping. (std::array<float, 16>)
#input_stream: "MATRIX:transform_matrix"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
#output_stream: "DETECTIONS:detections"
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 1
min_scale: 0.1484375
max_scale: 0.75
anchor_offset_x: 0.5
anchor_offset_y: 0.5
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
option_value: "input_size_width:tensor_width"
option_value: "input_size_height:tensor_height"
option_value: "num_layers:num_layers"
option_value: "strides:strides"
option_value: "interpolated_scale_aspect_ratio:interpolated_scale_aspect_ratio"
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
}
}
option_value: "num_boxes:num_boxes"
option_value: "x_scale:x_scale"
option_value: "y_scale:y_scale"
option_value: "h_scale:h_scale"
option_value: "w_scale:w_scale"
option_value: "min_score_thresh:min_score_thresh"
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Projects the detections from input tensor to the corresponding locations on
# the original image (input to the graph).
node {
calculator: "DetectionProjectionCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "PROJECTION_MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection.proto
+59
View File
@@ -0,0 +1,59 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/calculators/tensor/inference_calculator.proto";
import "mediapipe/framework/calculator_options.proto";
import "mediapipe/gpu/gpu_origin.proto";
option java_package = "com.google.mediapipe.modules.facedetection";
option java_outer_classname = "FaceDetectionFrontProto";
// Defines the face geometry pipeline estimation result format.
message FaceDetectionOptions {
extend mediapipe.CalculatorOptions {
optional FaceDetectionOptions ext = 374290926;
}
// Path to the TF Lite model (ex: /path/to/modelname.tflite).
optional string model_path = 1;
// The coordinate origin corner, either CONVENTIONAL or TOP_LEFT.
optional GpuOrigin.Mode gpu_origin = 11;
// Size of the tensor provided to the face-detection model.
optional int32 tensor_width = 21;
optional int32 tensor_height = 22;
// Number of output feature maps to generate the anchors on.
optional int32 num_layers = 23;
// Strides of each output feature maps.
repeated int32 strides = 24;
// The aspect ratio of the interpolated anchor from the SsdAnchorsCalculator.
optional float interpolated_scale_aspect_ratio = 25 [default = 1.0];
// The number of output boxes predicted by the detection model.
optional int32 num_boxes = 31;
// Parameters for decoding SSD detection model.
optional float x_scale = 32 [default = 0.0];
optional float y_scale = 33 [default = 0.0];
optional float w_scale = 34 [default = 0.0];
optional float h_scale = 35 [default = 0.0];
// Score threshold for perserving from the SSD detections.
optional float min_score_thresh = 36;
// TfLite delegate to run inference.
optional InferenceCalculatorOptions.Delegate delegate = 6;
}
mediapipe/modules/face_detection/face_detection_full_range.pbtxt
+54
View File
@@ -0,0 +1,54 @@
# MediaPipe graph to detect faces. (CPU input and inference by default.)
#
# It is required that "face_detection_full_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_full_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRange"
# input_stream: "IMAGE:image_frame"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFullRange"
# The input image, either ImageFrame, GpuBuffer, or (multi-backend) Image.
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where faces should be
# detected. (NormalizedRect)
input_stream: "ROI:roi"
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:detections"
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
node {
calculator: "FaceDetection"
input_stream: "IMAGE:image"
input_stream: "ROI:roi"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {
model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
tensor_width: 192
tensor_height: 192
num_layers: 1
strides: 4
interpolated_scale_aspect_ratio: 0.0
num_boxes: 2304
x_scale: 192.0
y_scale: 192.0
h_scale: 192.0
w_scale: 192.0
min_score_thresh: 0.6
}
}
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_full_range_common.pbtxt
-102
View File
@@ -1,102 +0,0 @@
# MediaPipe graph performing common processing to detect faces using
# face_detection_full_range_sparse.tflite model, currently consisting of tensor
# post processing.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFullRangeCommon"
# input_stream: "TENSORS:detection_tensors"
# input_stream: "MATRIX:transform_matrix"
# output_stream: "DETECTIONS:detections"
# }
type: "FaceDetectionShortRangeCommon"
# Detection tensors. (std::vector<Tensor>)
input_stream: "TENSORS:detection_tensors"
# A 4x4 row-major-order matrix that maps a point represented in the detection
# tensors to a desired coordinate system, e.g., in the original input image
# before scaling/cropping. (std::array<float, 16>)
input_stream: "MATRIX:transform_matrix"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 1
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 192
input_size_width: 192
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 4
aspect_ratios: 1.0
fixed_anchor_size: true
interpolated_scale_aspect_ratio: 0.0
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 2304
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 192.0
y_scale: 192.0
h_scale: 192.0
w_scale: 192.0
min_score_thresh: 0.6
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Projects the detections from input tensor to the corresponding locations on
# the original image (input to the graph).
node {
calculator: "DetectionProjectionCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "PROJECTION_MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection_full_range_cpu.pbtxt
+12 -67
View File
@@ -1,80 +1,25 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_full_range_sparse.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFullRangeCpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
# MediaPipe graph to detect faces. (CPU input and inference.)
type: "FaceDetectionFullRangeCpu"
# CPU image. (ImageFrame)
# The input image, either ImageFrame, or (multi-backend) Image.
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input CPU image (ImageFrame) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "IMAGE:multi_backend_image"
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
# Transforms the input image into a 192x192 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
}
}
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
delegate {
xnnpack {}
}
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFullRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
calculator: "FaceDetectionFullRange"
input_stream: "IMAGE:image"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {
delegate { xnnpack {} }
}
}
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_full_range_gpu.pbtxt
+11 -65
View File
@@ -1,80 +1,26 @@
# MediaPipe graph to detect faces. (GPU input, and inference is executed on
# GPU.)
#
# It is required that "face_detection_full_range_sparse.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFullRangeGpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
# MediaPipe graph to detect faces. (GPU input and inference.)
type: "FaceDetectionFullRangeGpu"
# GPU image. (GpuBuffer)
# The input image, either GpuBuffer, or (multi-backend) Image.
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input GPU image (GpuBuffer) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "IMAGE:multi_backend_image"
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: TOP_LEFT
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
#
calculator: "FaceDetectionFullRange"
input_stream: "IMAGE:image"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {
gpu_origin: TOP_LEFT
delegate: { gpu { use_advanced_gpu_api: true } }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFullRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_full_range_image.pbtxt
+6 -45
View File
@@ -36,51 +36,12 @@ node {
}
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
node {
calculator: "FaceDetectionFullRange"
input_stream: "IMAGE:throttled_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: CONVENTIONAL
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
# TODO: Use GraphOptions to modify the delegate field to be
# `delegate { xnnpack {} }` for the CPU only use cases.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
#
delegate: { gpu { use_advanced_gpu_api: true } }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFullRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_short_range.pbtxt
+57
View File
@@ -0,0 +1,57 @@
# MediaPipe graph to detect faces. (CPU input and inference by default.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRange"
# input_stream: "IMAGE:image_frame"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionShortRange"
# The input image, either ImageFrame, GpuBuffer, or (multi-backend) Image.
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where faces should be
# detected. (NormalizedRect)
input_stream: "ROI:roi"
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:detections"
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
node {
calculator: "FaceDetection"
input_stream: "IMAGE:image"
input_stream: "ROI:roi"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
tensor_width: 128
tensor_height: 128
num_layers: 4
strides: 8
strides: 16
strides: 16
strides: 16
interpolated_scale_aspect_ratio: 1.0
num_boxes: 896
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.5
}
}
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_short_range_by_roi_cpu.pbtxt
+12 -65
View File
@@ -1,21 +1,8 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeByRoiCpu"
# input_stream: "IMAGE:image"
# input_stream: "ROI:roi"
# output_stream: "DETECTIONS:face_detections"
# }
# MediaPipe graph to detect faces. (CPU input and inference, with region-of-interest.)
type: "FaceDetectionShortRangeByRoiCpu"
type: "FaceDetectionShortRangeCpu"
# CPU image. (ImageFrame)
# The input image, either ImageFrame, or (multi-backend) Image.
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where faces should be
@@ -23,61 +10,21 @@ input_stream: "IMAGE:image"
input_stream: "ROI:roi"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input CPU image (ImageFrame) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "IMAGE:multi_backend_image"
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
# Transforms specified region of image into 128x128 tensor keeping aspect ratio
# (padding tensor if needed).
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
input_stream: "NORM_RECT:roi"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
}
}
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
calculator: "FaceDetectionShortRange"
input_stream: "IMAGE:image"
input_stream: "ROI:roi"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {
delegate { xnnpack {} }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_short_range_by_roi_gpu.pbtxt
+15 -67
View File
@@ -1,21 +1,8 @@
# MediaPipe graph to detect faces. (GPU input, and inference is executed on
# GPU.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeByRoiGpu"
# input_stream: "IMAGE:image"
# input_stream: "ROI:roi"
# output_stream: "DETECTIONS:face_detections"
# }
# MediaPipe graph to detect faces. (GPU input and inference, with region-of-interest.)
type: "FaceDetectionShortRangeByRoiGpu"
type: "FaceDetectionShortRangeGpu"
# GPU image. (GpuBuffer)
# The input image, either ImageFrame, or (multi-backend) Image.
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where faces should be
@@ -23,61 +10,22 @@ input_stream: "IMAGE:image"
input_stream: "ROI:roi"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input GPU image (GpuBuffer) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "IMAGE:multi_backend_image"
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
# Transforms specified region of image into 128x128 tensor keeping aspect ratio
# (padding tensor if needed).
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
input_stream: "NORM_RECT:roi"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: TOP_LEFT
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
calculator: "FaceDetectionShortRange"
input_stream: "IMAGE:image"
input_stream: "ROI:roi"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {
gpu_origin: TOP_LEFT
delegate: { gpu { use_advanced_gpu_api: true } }
}
}
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_short_range_common.pbtxt
-103
View File
@@ -1,103 +0,0 @@
# MediaPipe graph performing common processing to detect faces, currently
# consisting of tensor post processing.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeCommon"
# input_stream: "TENSORS:detection_tensors"
# input_stream: "MATRIX:transform_matrix"
# output_stream: "DETECTIONS:detections"
# }
type: "FaceDetectionShortRangeCommon"
# Detection tensors. (std::vector<Tensor>)
input_stream: "TENSORS:detection_tensors"
# A 4x4 row-major-order matrix that maps a point represented in the detection
# tensors to a desired coordinate system, e.g., in the original input image
# before scaling/cropping. (std::array<float, 16>)
input_stream: "MATRIX:transform_matrix"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 4
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 128
input_size_width: 128
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 8
strides: 16
strides: 16
strides: 16
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 896
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.5
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Projects the detections from input tensor to the corresponding locations on
# the original image (input to the graph).
node {
calculator: "DetectionProjectionCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "PROJECTION_MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection_short_range_cpu.pbtxt
+10 -63
View File
@@ -1,78 +1,25 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeCpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
# MediaPipe graph to detect faces. (CPU input and inference.)
type: "FaceDetectionShortRangeCpu"
# CPU image. (ImageFrame)
# The input image, either ImageFrame, or (multi-backend) Image.
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input CPU image (ImageFrame) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "IMAGE:multi_backend_image"
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
}
}
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
calculator: "FaceDetectionShortRange"
input_stream: "IMAGE:image"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {
delegate { xnnpack {} }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt
+13 -65
View File
@@ -1,78 +1,26 @@
# MediaPipe graph to detect faces. (GPU input, and inference is executed on
# GPU.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeGpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
# MediaPipe graph to detect faces. (GPU input and inference.)
type: "FaceDetectionShortRangeGpu"
# GPU image. (GpuBuffer)
# The input image, either GpuBuffer, or (multi-backend) Image.
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input GPU image (GpuBuffer) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "IMAGE:multi_backend_image"
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: TOP_LEFT
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
calculator: "FaceDetectionShortRange"
input_stream: "IMAGE:image"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {
gpu_origin: TOP_LEFT
delegate: { gpu { use_advanced_gpu_api: true } }
}
}
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_short_range_image.pbtxt
+6 -46
View File
@@ -43,52 +43,12 @@ node {
}
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
node {
calculator: "FaceDetectionShortRange"
input_stream: "IMAGE:throttled_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: CONVENTIONAL
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
# TODO: Use GraphOptions to modify the delegate field to be
# `delegate { xnnpack {} }` for the CPU only use cases.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
#
delegate: { gpu { use_advanced_gpu_api: true } }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
option_value: "OPTIONS:options"
}
mediapipe/modules/face_detection/face_detection_test.cc
+384
View File
@@ -0,0 +1,384 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <vector>
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h"
#include "mediapipe/calculators/tflite/ssd_anchors_calculator.pb.h"
#include "mediapipe/calculators/util/non_max_suppression_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/options_util.h"
#include "mediapipe/framework/tool/test_util.h"
#include "mediapipe/gpu/gpu_origin.pb.h"
#include "mediapipe/modules/face_detection/face_detection.pb.h"
#if !defined(__APPLE__) && !__ANDROID__
#include "mediapipe/gpu/gl_app_texture_support.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_test_base.h"
#endif // !defined(__APPLE__) && !__ANDROID__
namespace mediapipe {
namespace {
using mediapipe::FaceDetectionOptions;
// Ensure protobuf registration.
void RegisterProtobufTypes() {
MakePacket<mediapipe::InferenceCalculatorOptions::Delegate>();
MakePacket<mediapipe::FaceDetectionOptions>();
MakePacket<mediapipe::InferenceCalculatorOptions>();
MakePacket<mediapipe::ImageToTensorCalculatorOptions>();
MakePacket<mediapipe::SsdAnchorsCalculatorOptions>();
MakePacket<mediapipe::TensorsToDetectionsCalculatorOptions>();
MakePacket<mediapipe::NonMaxSuppressionCalculatorOptions>();
}
// Returns a Packet with an ImageFrame showing a face.
Packet TestImageFrame() {
std::unique_ptr<ImageFrame> input_image = LoadTestPng(
file::JoinPath(GetTestRootDir(), "mediapipe/objc/testdata/sergey.png"));
EXPECT_EQ(input_image->Height(), 600);
return MakePacket<ImageFrame>(std::move(*input_image));
}
// Returns the registered type name for the basic face-detection-graph.
std::string GetFaceDetectionGraphType() { return "FaceDetectionWithoutRoi"; }
// Returns the config from "face_detection_without_roi.pbtxt".
CalculatorGraphConfig GetFaceDetectionGraph() {
return GraphRegistry().CreateByName("", GetFaceDetectionGraphType()).value();
}
// Returns the config from "face_detection.pbtxt".
CalculatorGraphConfig GetFaceDetectionWithRoiGraph() {
return GraphRegistry().CreateByName("", "FaceDetection").value();
}
// Returns the config from "face_detection_short_range.pbtxt".
CalculatorGraphConfig GetFaceDetectionShortRangeCpu() {
CalculatorGraphConfig config =
GraphRegistry().CreateByName("", "FaceDetectionShortRangeCpu").value();
return config;
}
// Returns the FaceDetectionOptions from "face_detection_short_range_cpu.pbtxt".
FaceDetectionOptions GetFaceDetectionShortRangeOptions() {
CalculatorGraphConfig config;
LoadTestGraph(&config,
GetTestFilePath("mediapipe/modules/face_detection/"
"face_detection_short_range.binarypb"));
tool::OptionsMap map;
map.Initialize(config.node(0));
return map.Get<FaceDetectionOptions>();
}
// Returns the FaceDetectionOptions from "face_detection_full_range_cpu.pbtxt".
FaceDetectionOptions GetFaceDetectionFullRangeOptions() {
CalculatorGraphConfig config;
LoadTestGraph(&config, GetTestFilePath("mediapipe/modules/face_detection/"
"face_detection_full_range.binarypb"));
tool::OptionsMap map;
map.Initialize(config.node(0));
return map.Get<FaceDetectionOptions>();
}
// Returns the FaceDetectionOptions needed to enable CPU processing.
FaceDetectionOptions GetCpuOptions() {
FaceDetectionOptions result;
result.mutable_delegate()->xnnpack();
return result;
}
// Returns the FaceDetectionOptions needed to enable GPU processing.
FaceDetectionOptions GetGpuOptions() {
FaceDetectionOptions result;
result.set_gpu_origin(mediapipe::GpuOrigin_Mode::GpuOrigin_Mode_TOP_LEFT);
result.mutable_delegate()->mutable_gpu()->set_use_advanced_gpu_api(true);
return result;
}
// Returns an example region of interest rectangle.
mediapipe::NormalizedRect GetTestRoi() {
mediapipe::NormalizedRect result;
result.set_x_center(0.5);
result.set_y_center(0.5);
result.set_width(0.8);
result.set_height(0.8);
return result;
}
// Tests for options input and output packets and streams.
class FaceDetectionTest : public ::testing::Test {
protected:
void SetUp() override { RegisterProtobufTypes(); }
void TearDown() override {}
};
TEST_F(FaceDetectionTest, ExpandFaceDetectionShortRangeCpu) {
CalculatorGraphConfig config = GetFaceDetectionShortRangeCpu();
Packet frame1 = TestImageFrame();
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(config));
std::vector<Packet> output;
MP_ASSERT_OK(graph.ObserveOutputStream("detections", [&](const Packet& p) {
output.push_back(p);
return absl::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("image", frame1.At(Timestamp(20000))));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_EXPECT_OK(graph.WaitUntilDone());
ASSERT_EQ(output.size(), 1);
EXPECT_EQ(output.front().Get<std::vector<mediapipe::Detection>>().size(), 1);
}
TEST_F(FaceDetectionTest, ExpandFaceDetection) {
CalculatorGraphConfig config = GetFaceDetectionGraph();
mediapipe::FaceDetectionOptions face_options =
GetFaceDetectionShortRangeOptions();
face_options.MergeFrom(GetCpuOptions());
config.clear_graph_options();
config.add_graph_options()->PackFrom(face_options);
Packet frame1 = TestImageFrame();
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(config));
std::vector<Packet> output;
MP_ASSERT_OK(graph.ObserveOutputStream("detections", [&](const Packet& p) {
output.push_back(p);
return absl::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("image", frame1.At(Timestamp(20000))));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_EXPECT_OK(graph.WaitUntilDone());
ASSERT_EQ(output.size(), 1);
EXPECT_EQ(output.front().Get<std::vector<mediapipe::Detection>>().size(), 1);
}
TEST_F(FaceDetectionTest, FaceDetectionShortRangeApi) {
CalculatorGraphConfig config = GetFaceDetectionGraph();
config.clear_graph_options();
mediapipe::FaceDetectionOptions face_options =
GetFaceDetectionShortRangeOptions();
Subgraph::SubgraphOptions graph_options;
face_options.MergeFrom(GetCpuOptions());
graph_options.add_node_options()->PackFrom(face_options);
Packet frame1 = TestImageFrame();
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize({config}, {}, {}, GetFaceDetectionGraphType(),
&graph_options));
std::vector<Packet> output;
MP_ASSERT_OK(graph.ObserveOutputStream("detections", [&](const Packet& p) {
output.push_back(p);
return absl::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("image", frame1.At(Timestamp(20000))));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_EXPECT_OK(graph.WaitUntilDone());
ASSERT_EQ(output.size(), 1);
EXPECT_EQ(output.front().Get<std::vector<mediapipe::Detection>>().size(), 1);
}
TEST_F(FaceDetectionTest, FaceDetectionWrapperApi) {
CalculatorGraphConfig config = GetFaceDetectionGraph();
config.clear_graph_options();
mediapipe::FaceDetectionOptions face_options =
GetFaceDetectionShortRangeOptions();
face_options.MergeFrom(GetCpuOptions());
Subgraph::SubgraphOptions graph_options;
graph_options.add_node_options()->PackFrom(face_options);
Packet frame1 = TestImageFrame();
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize({config}, {}, {}, GetFaceDetectionGraphType(),
&graph_options));
std::vector<Packet> output;
MP_ASSERT_OK(graph.ObserveOutputStream("detections", [&](const Packet& p) {
output.push_back(p);
return absl::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("image", frame1.At(Timestamp(20000))));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_EXPECT_OK(graph.WaitUntilDone());
ASSERT_EQ(output.size(), 1);
EXPECT_EQ(output.front().Get<std::vector<mediapipe::Detection>>().size(), 1);
}
TEST_F(FaceDetectionTest, FaceDetectionFullRangeApi) {
CalculatorGraphConfig config = GetFaceDetectionGraph();
config.clear_graph_options();
mediapipe::FaceDetectionOptions face_options =
GetFaceDetectionFullRangeOptions();
Subgraph::SubgraphOptions graph_options;
face_options.MergeFrom(GetCpuOptions());
graph_options.add_node_options()->PackFrom(face_options);
Packet frame1 = TestImageFrame();
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize({config}, {}, {}, GetFaceDetectionGraphType(),
&graph_options));
std::vector<Packet> output;
MP_ASSERT_OK(graph.ObserveOutputStream("detections", [&](const Packet& p) {
output.push_back(p);
return absl::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("image", frame1.At(Timestamp(20000))));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_EXPECT_OK(graph.WaitUntilDone());
ASSERT_EQ(output.size(), 1);
EXPECT_EQ(output.front().Get<std::vector<mediapipe::Detection>>().size(), 1);
}
TEST_F(FaceDetectionTest, FaceDetectionShortRangeByRoiCpu) {
CalculatorGraphConfig config = GetFaceDetectionWithRoiGraph();
config.clear_graph_options();
mediapipe::FaceDetectionOptions face_options =
GetFaceDetectionShortRangeOptions();
face_options.MergeFrom(GetCpuOptions());
Subgraph::SubgraphOptions graph_options;
graph_options.add_node_options()->PackFrom(face_options);
Packet frame1 = TestImageFrame();
CalculatorGraph graph;
MP_ASSERT_OK(
graph.Initialize({config}, {}, {}, "FaceDetection", &graph_options));
std::vector<Packet> output;
MP_ASSERT_OK(graph.ObserveOutputStream("detections", [&](const Packet& p) {
output.push_back(p);
return absl::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("image", frame1.At(Timestamp(20000))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"roi", MakePacket<mediapipe::NormalizedRect>(GetTestRoi())
.At(Timestamp(20000))));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_EXPECT_OK(graph.WaitUntilDone());
ASSERT_EQ(output.size(), 1);
EXPECT_EQ(output.front().Get<std::vector<mediapipe::Detection>>().size(), 1);
}
// These GpuBuffer tests are disabled on mobile for now.
#if !defined(__APPLE__) && !__ANDROID__
class FaceDetectionGpuTest : public mediapipe::GpuTestBase {
protected:
void SetUp() override {}
void TearDown() override {}
// Returns a Packet with a GpuBuffer from an ImageFrame.
Packet GpuBuffer(Packet image_frame) {
std::unique_ptr<mediapipe::GpuBuffer> gpu_buffer;
helper_.RunInGlContext([this, &image_frame, &gpu_buffer] {
auto src = helper_.CreateSourceTexture(image_frame.Get<ImageFrame>());
gpu_buffer = src.GetFrame<mediapipe::GpuBuffer>();
});
return Adopt(gpu_buffer.release());
}
};
TEST_F(FaceDetectionGpuTest, FaceDetectionFullRangeGpu) {
CalculatorGraphConfig config = GetFaceDetectionGraph();
config.clear_graph_options();
mediapipe::FaceDetectionOptions face_options =
GetFaceDetectionFullRangeOptions();
face_options.MergeFrom(GetGpuOptions());
Subgraph::SubgraphOptions graph_options;
graph_options.add_node_options()->PackFrom(face_options);
Packet frame1 = GpuBuffer(TestImageFrame());
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize({config}, {}, {}, GetFaceDetectionGraphType(),
&graph_options));
MP_ASSERT_OK(mediapipe::SetExternalGlContextForGraph(
&graph, helper_.GetGlContext().native_context()));
std::vector<Packet> output;
MP_ASSERT_OK(graph.ObserveOutputStream("detections", [&](const Packet& p) {
output.push_back(p);
return absl::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("image", frame1.At(Timestamp(20000))));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_EXPECT_OK(graph.WaitUntilDone());
ASSERT_EQ(output.size(), 1);
EXPECT_EQ(output.front().Get<std::vector<mediapipe::Detection>>().size(), 1);
}
TEST_F(FaceDetectionGpuTest, FaceDetectionShortRangeGpu) {
CalculatorGraphConfig config = GetFaceDetectionGraph();
config.clear_graph_options();
mediapipe::FaceDetectionOptions face_options =
GetFaceDetectionShortRangeOptions();
face_options.MergeFrom(GetGpuOptions());
Subgraph::SubgraphOptions graph_options;
graph_options.add_node_options()->PackFrom(face_options);
Packet frame1 = GpuBuffer(TestImageFrame());
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize({config}, {}, {}, GetFaceDetectionGraphType(),
&graph_options));
MP_ASSERT_OK(mediapipe::SetExternalGlContextForGraph(
&graph, helper_.GetGlContext().native_context()));
std::vector<Packet> output;
MP_ASSERT_OK(graph.ObserveOutputStream("detections", [&](const Packet& p) {
output.push_back(p);
return absl::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("image", frame1.At(Timestamp(20000))));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_EXPECT_OK(graph.WaitUntilDone());
ASSERT_EQ(output.size(), 1);
EXPECT_EQ(output.front().Get<std::vector<mediapipe::Detection>>().size(), 1);
}
#endif // #if !defined(__APPLE__) && !__ANDROID__
} // namespace
} // namespace mediapipe
mediapipe/modules/face_detection/face_detection_without_roi.pbtxt
+35
View File
@@ -0,0 +1,35 @@
# MediaPipe graph to detect faces.
# This graph omits the "ROI" input stream of the FaceDetection graph.
# For now top level graph input streams can only be ommitted using an
# enclosing graph, see b/202896911.
# TODO: Remove this graph after b/202896911 is addressed.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFrontCpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionWithoutRoi"
# The input image, either ImageFrame, GpuBuffer, or (multi-backend) Image.
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:detections"
# The face detection graph options.
graph_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
node {
calculator: "FaceDetection"
input_stream: "IMAGE:image"
output_stream: "DETECTIONS:detections"
node_options: {
[type.googleapis.com/mediapipe.FaceDetectionOptions] {}
}
option_value: "OPTIONS:options"
}