Project import generated by Copybara.
GitOrigin-RevId: d073f8e21be2fcc0e503cb97c6695078b6b75310
This commit is contained in:
MediaPipe Team
@@ -28,8 +28,9 @@ cc_library(
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:immediate_mux_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/graphs/face_effect/subgraphs:single_face_smooth_landmark_gpu",
|
||||
"//mediapipe/modules/face_geometry",
|
||||
"//mediapipe/framework/tool:switch_container",
|
||||
"//mediapipe/graphs/face_effect/subgraphs:single_face_geometry_from_detection_gpu",
|
||||
"//mediapipe/graphs/face_effect/subgraphs:single_face_geometry_from_landmarks_gpu",
|
||||
"//mediapipe/modules/face_geometry:effect_renderer_calculator",
|
||||
"//mediapipe/modules/face_geometry:env_generator_calculator",
|
||||
],
|
||||
|
||||
@@ -18,6 +18,16 @@ licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
encode_binary_proto(
|
||||
name = "axis",
|
||||
input = "axis.pbtxt",
|
||||
message_type = "mediapipe.face_geometry.Mesh3d",
|
||||
output = "axis.binarypb",
|
||||
deps = [
|
||||
"//mediapipe/modules/face_geometry/protos:mesh_3d_proto",
|
||||
],
|
||||
)
|
||||
|
||||
encode_binary_proto(
|
||||
name = "glasses",
|
||||
input = "glasses.pbtxt",
|
||||
@@ -31,6 +41,7 @@ encode_binary_proto(
|
||||
# `.pngblob` is used instead of `.png` to prevent iOS build from preprocessing the image.
|
||||
# OpenCV is unable to read a PNG file preprocessed by the iOS build.
|
||||
exports_files([
|
||||
"axis.pngblob",
|
||||
"facepaint.pngblob",
|
||||
"glasses.pngblob",
|
||||
])
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
vertex_type: VERTEX_PT
|
||||
primitive_type: TRIANGLE
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 11.500000
|
||||
vertex_buffer: 0.873006
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 11.500000
|
||||
vertex_buffer: 0.928502
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 11.500000
|
||||
vertex_buffer: 0.928502
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 11.500000
|
||||
vertex_buffer: 0.873006
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.500000
|
||||
vertex_buffer: 0.928502
|
||||
vertex_buffer: 0.500000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.500000
|
||||
vertex_buffer: 0.873006
|
||||
vertex_buffer: 0.500000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.500000
|
||||
vertex_buffer: 0.928502
|
||||
vertex_buffer: 0.250000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.500000
|
||||
vertex_buffer: 0.873006
|
||||
vertex_buffer: 0.250000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 11.500000
|
||||
vertex_buffer: 0.928502
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 11.500000
|
||||
vertex_buffer: 0.873006
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.500000
|
||||
vertex_buffer: 0.983999
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.500000
|
||||
vertex_buffer: 0.983999
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.500000
|
||||
vertex_buffer: 0.817509
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.500000
|
||||
vertex_buffer: 0.817509
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.069341
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.123429
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.123429
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.069341
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.123419
|
||||
vertex_buffer: 0.499992
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.069341
|
||||
vertex_buffer: 0.500000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.123429
|
||||
vertex_buffer: 0.250000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.069341
|
||||
vertex_buffer: 0.250000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.123429
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.069341
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.177516
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.177516
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.015254
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.015254
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.472252
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.527748
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.527748
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.472252
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.527748
|
||||
vertex_buffer: 0.500000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.472252
|
||||
vertex_buffer: 0.500000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.527748
|
||||
vertex_buffer: 0.250000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.472252
|
||||
vertex_buffer: 0.250000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.527748
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.472252
|
||||
vertex_buffer: 0.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.583245
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: 0.100000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.583245
|
||||
vertex_buffer: 0.750000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 8.400000
|
||||
vertex_buffer: 0.416755
|
||||
vertex_buffer: 1.000000
|
||||
vertex_buffer: -0.100000
|
||||
vertex_buffer: 3.000000
|
||||
vertex_buffer: 8.600000
|
||||
vertex_buffer: 0.416755
|
||||
vertex_buffer: 0.750000
|
||||
index_buffer: 0
|
||||
index_buffer: 1
|
||||
index_buffer: 2
|
||||
index_buffer: 0
|
||||
index_buffer: 2
|
||||
index_buffer: 3
|
||||
index_buffer: 3
|
||||
index_buffer: 2
|
||||
index_buffer: 4
|
||||
index_buffer: 3
|
||||
index_buffer: 4
|
||||
index_buffer: 5
|
||||
index_buffer: 5
|
||||
index_buffer: 4
|
||||
index_buffer: 6
|
||||
index_buffer: 5
|
||||
index_buffer: 6
|
||||
index_buffer: 7
|
||||
index_buffer: 7
|
||||
index_buffer: 6
|
||||
index_buffer: 8
|
||||
index_buffer: 7
|
||||
index_buffer: 8
|
||||
index_buffer: 9
|
||||
index_buffer: 1
|
||||
index_buffer: 10
|
||||
index_buffer: 11
|
||||
index_buffer: 1
|
||||
index_buffer: 11
|
||||
index_buffer: 2
|
||||
index_buffer: 12
|
||||
index_buffer: 0
|
||||
index_buffer: 3
|
||||
index_buffer: 12
|
||||
index_buffer: 3
|
||||
index_buffer: 13
|
||||
index_buffer: 14
|
||||
index_buffer: 15
|
||||
index_buffer: 16
|
||||
index_buffer: 14
|
||||
index_buffer: 16
|
||||
index_buffer: 17
|
||||
index_buffer: 17
|
||||
index_buffer: 16
|
||||
index_buffer: 18
|
||||
index_buffer: 17
|
||||
index_buffer: 18
|
||||
index_buffer: 19
|
||||
index_buffer: 19
|
||||
index_buffer: 18
|
||||
index_buffer: 20
|
||||
index_buffer: 19
|
||||
index_buffer: 20
|
||||
index_buffer: 21
|
||||
index_buffer: 21
|
||||
index_buffer: 20
|
||||
index_buffer: 22
|
||||
index_buffer: 21
|
||||
index_buffer: 22
|
||||
index_buffer: 23
|
||||
index_buffer: 15
|
||||
index_buffer: 24
|
||||
index_buffer: 25
|
||||
index_buffer: 15
|
||||
index_buffer: 25
|
||||
index_buffer: 16
|
||||
index_buffer: 26
|
||||
index_buffer: 14
|
||||
index_buffer: 17
|
||||
index_buffer: 26
|
||||
index_buffer: 17
|
||||
index_buffer: 27
|
||||
index_buffer: 28
|
||||
index_buffer: 29
|
||||
index_buffer: 30
|
||||
index_buffer: 28
|
||||
index_buffer: 30
|
||||
index_buffer: 31
|
||||
index_buffer: 31
|
||||
index_buffer: 30
|
||||
index_buffer: 32
|
||||
index_buffer: 31
|
||||
index_buffer: 32
|
||||
index_buffer: 33
|
||||
index_buffer: 33
|
||||
index_buffer: 32
|
||||
index_buffer: 34
|
||||
index_buffer: 33
|
||||
index_buffer: 34
|
||||
index_buffer: 35
|
||||
index_buffer: 35
|
||||
index_buffer: 34
|
||||
index_buffer: 36
|
||||
index_buffer: 35
|
||||
index_buffer: 36
|
||||
index_buffer: 37
|
||||
index_buffer: 29
|
||||
index_buffer: 38
|
||||
index_buffer: 39
|
||||
index_buffer: 29
|
||||
index_buffer: 39
|
||||
index_buffer: 30
|
||||
index_buffer: 40
|
||||
index_buffer: 28
|
||||
index_buffer: 31
|
||||
index_buffer: 40
|
||||
index_buffer: 31
|
||||
index_buffer: 41
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 492 B |
@@ -3,11 +3,20 @@
|
||||
# GPU buffer. (GpuBuffer)
|
||||
input_stream: "input_video"
|
||||
|
||||
# Boolean flag, which indicates whether the Facepaint effect is selected. (bool)
|
||||
# An integer, which indicate which effect is selected. (int)
|
||||
#
|
||||
# If `true`, the Facepaint effect will be rendered.
|
||||
# If `false`, the Glasses effect will be rendered.
|
||||
input_stream: "is_facepaint_effect_selected"
|
||||
# If `selected_effect_id` is `0`, the Axis effect is selected.
|
||||
# If `selected_effect_id` is `1`, the Facepaint effect is selected.
|
||||
# If `selected_effect_id` is `2`, the Glasses effect is selected.
|
||||
#
|
||||
# No other values are allowed for `selected_effect_id`.
|
||||
input_stream: "selected_effect_id"
|
||||
|
||||
# Indicates whether to use the face detection as the input source. (bool)
|
||||
#
|
||||
# If `true`, the face detection pipeline will be used to produce landmarks.
|
||||
# If `false`, the face landmark pipeline will be used to produce landmarks.
|
||||
input_side_packet: "use_face_detection_input_source"
|
||||
|
||||
# Output image with rendered results. (GpuBuffer)
|
||||
output_stream: "output_video"
|
||||
@@ -59,87 +68,63 @@ node {
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detects a single face and corresponding landmarks. The landmarks
|
||||
# are also "smoothed" to achieve better visual results.
|
||||
# Computes the face geometry for a single face. The input source is defined
|
||||
# through `use_face_detection_input_source`.
|
||||
node {
|
||||
calculator: "SingleFaceSmoothLandmarkGpu"
|
||||
calculator: "SwitchContainer"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
output_stream: "LANDMARKS:multi_face_landmarks"
|
||||
}
|
||||
|
||||
# Extracts the throttled input video frame dimensions as a separate packet.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:throttled_input_video"
|
||||
output_stream: "SIZE:input_video_size"
|
||||
}
|
||||
|
||||
# Subgraph that computes face geometry from landmarks for a single face.
|
||||
node {
|
||||
calculator: "FaceGeometry"
|
||||
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
|
||||
input_stream: "IMAGE_SIZE:input_video_size"
|
||||
input_side_packet: "ENABLE:use_face_detection_input_source"
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
|
||||
}
|
||||
|
||||
# Decides whether to render the Facepaint effect based on the
|
||||
# `is_facepaint_effect_selected` flag value.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "throttled_input_video"
|
||||
input_stream: "multi_face_geometry"
|
||||
input_stream: "ALLOW:is_facepaint_effect_selected"
|
||||
output_stream: "facepaint_effect_throttled_input_video"
|
||||
output_stream: "facepaint_effect_multi_face_geometry"
|
||||
}
|
||||
|
||||
# Renders the Facepaint effect.
|
||||
node {
|
||||
calculator: "FaceGeometryEffectRendererCalculator"
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
input_stream: "IMAGE_GPU:facepaint_effect_throttled_input_video"
|
||||
input_stream: "MULTI_FACE_GEOMETRY:facepaint_effect_multi_face_geometry"
|
||||
output_stream: "IMAGE_GPU:facepaint_effect_output_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
|
||||
effect_texture_path: "mediapipe/graphs/face_effect/data/facepaint.pngblob"
|
||||
[type.googleapis.com/mediapipe.SwitchContainerOptions] {
|
||||
contained_node: {
|
||||
calculator: "SingleFaceGeometryFromLandmarksGpu"
|
||||
}
|
||||
contained_node: {
|
||||
calculator: "SingleFaceGeometryFromDetectionGpu"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decides whether to render the Glasses effect based on the
|
||||
# `is_facepaint_effect_selected` flag value.
|
||||
# Renders the selected effect based on `selected_effect_id`.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "throttled_input_video"
|
||||
input_stream: "multi_face_geometry"
|
||||
input_stream: "DISALLOW:is_facepaint_effect_selected"
|
||||
output_stream: "glasses_effect_throttled_input_video"
|
||||
output_stream: "glasses_effect_multi_face_geometry"
|
||||
}
|
||||
|
||||
# Renders the Glasses effect.
|
||||
node {
|
||||
calculator: "FaceGeometryEffectRendererCalculator"
|
||||
calculator: "SwitchContainer"
|
||||
input_stream: "SELECT:selected_effect_id"
|
||||
input_stream: "IMAGE_GPU:throttled_input_video"
|
||||
input_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
input_stream: "IMAGE_GPU:glasses_effect_throttled_input_video"
|
||||
input_stream: "MULTI_FACE_GEOMETRY:glasses_effect_multi_face_geometry"
|
||||
output_stream: "IMAGE_GPU:glasses_effect_output_video"
|
||||
output_stream: "IMAGE_GPU:output_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
|
||||
effect_texture_path: "mediapipe/graphs/face_effect/data/glasses.pngblob"
|
||||
effect_mesh_3d_path: "mediapipe/graphs/face_effect/data/glasses.binarypb"
|
||||
[type.googleapis.com/mediapipe.SwitchContainerOptions] {
|
||||
contained_node: {
|
||||
calculator: "FaceGeometryEffectRendererCalculator"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
|
||||
effect_texture_path: "mediapipe/graphs/face_effect/data/axis.pngblob"
|
||||
effect_mesh_3d_path: "mediapipe/graphs/face_effect/data/axis.binarypb"
|
||||
}
|
||||
}
|
||||
}
|
||||
contained_node: {
|
||||
calculator: "FaceGeometryEffectRendererCalculator"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
|
||||
effect_texture_path: "mediapipe/graphs/face_effect/data/facepaint.pngblob"
|
||||
}
|
||||
}
|
||||
}
|
||||
contained_node: {
|
||||
calculator: "FaceGeometryEffectRendererCalculator"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
|
||||
effect_texture_path: "mediapipe/graphs/face_effect/data/glasses.pngblob"
|
||||
effect_mesh_3d_path: "mediapipe/graphs/face_effect/data/glasses.binarypb"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decides which of the Facepaint or the Glasses rendered results should be sent
|
||||
# as the output GPU frame.
|
||||
node {
|
||||
calculator: "ImmediateMuxCalculator"
|
||||
input_stream: "facepaint_effect_output_video"
|
||||
input_stream: "glasses_effect_output_video"
|
||||
output_stream: "output_video"
|
||||
}
|
||||
|
||||
|
||||
@@ -22,15 +22,40 @@ licenses(["notice"])
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "single_face_smooth_landmark_gpu",
|
||||
graph = "single_face_smooth_landmark_gpu.pbtxt",
|
||||
register_as = "SingleFaceSmoothLandmarkGpu",
|
||||
name = "face_landmarks_smoothing",
|
||||
graph = "face_landmarks_smoothing.pbtxt",
|
||||
register_as = "FaceLandmarksSmoothing",
|
||||
deps = [
|
||||
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "single_face_geometry_from_detection_gpu",
|
||||
graph = "single_face_geometry_from_detection_gpu.pbtxt",
|
||||
register_as = "SingleFaceGeometryFromDetectionGpu",
|
||||
deps = [
|
||||
":face_landmarks_smoothing",
|
||||
"//mediapipe/calculators/core:concatenate_detection_vector_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/modules/face_detection:face_detection_front_gpu",
|
||||
"//mediapipe/modules/face_geometry:face_geometry_from_detection",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "single_face_geometry_from_landmarks_gpu",
|
||||
graph = "single_face_geometry_from_landmarks_gpu.pbtxt",
|
||||
register_as = "SingleFaceGeometryFromLandmarksGpu",
|
||||
deps = [
|
||||
":face_landmarks_smoothing",
|
||||
"//mediapipe/calculators/core:concatenate_vector_calculator",
|
||||
"//mediapipe/calculators/core:constant_side_packet_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
|
||||
"//mediapipe/modules/face_geometry:face_geometry_from_landmarks",
|
||||
"//mediapipe/modules/face_landmark:face_landmark_front_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# MediaPipe subgraph that smoothes face landmarks.
|
||||
|
||||
type: "FaceLandmarksSmoothing"
|
||||
|
||||
input_stream: "NORM_LANDMARKS:landmarks"
|
||||
input_stream: "IMAGE_SIZE:input_image_size"
|
||||
output_stream: "NORM_FILTERED_LANDMARKS:filtered_landmarks"
|
||||
|
||||
# Applies smoothing to a face landmark list. The filter options were handpicked
|
||||
# to achieve better visual results.
|
||||
node {
|
||||
calculator: "LandmarksSmoothingCalculator"
|
||||
input_stream: "NORM_LANDMARKS:landmarks"
|
||||
input_stream: "IMAGE_SIZE:input_image_size"
|
||||
output_stream: "NORM_FILTERED_LANDMARKS:filtered_landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
|
||||
velocity_filter: {
|
||||
window_size: 5
|
||||
velocity_scale: 20.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
# MediaPipe subgraph that extracts geometry from a single face using the face
|
||||
# landmark pipeline on an input GPU image. The face landmarks are also
|
||||
# "smoothed" to achieve better visual results.
|
||||
|
||||
type: "SingleFaceGeometryFromDetectionGpu"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "IMAGE:input_image"
|
||||
|
||||
# Environment that describes the current virtual scene.
|
||||
# (face_geometry::Environment)
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
|
||||
# A list of geometry data for a single detected face. The size of this
|
||||
# collection is at most 1 because of the single-face use in this graph.
|
||||
# (std::vector<face_geometry::FaceGeometry>)
|
||||
#
|
||||
# NOTE: if no face is detected at a particular timestamp, there will not be an
|
||||
# output packet in the `MULTI_FACE_GEOMETRY` stream for this timestamp. However,
|
||||
# the MediaPipe framework will internally inform the downstream calculators of
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
|
||||
|
||||
# Subgraph that detects faces and corresponding landmarks using the face
|
||||
# detection pipeline.
|
||||
node {
|
||||
calculator: "FaceDetectionFrontGpu"
|
||||
input_stream: "IMAGE:input_image"
|
||||
output_stream: "DETECTIONS:multi_face_detection"
|
||||
}
|
||||
|
||||
# Extracts the first face detection associated with the most prominent face from
|
||||
# a collection.
|
||||
node {
|
||||
calculator: "SplitDetectionVectorCalculator"
|
||||
input_stream: "multi_face_detection"
|
||||
output_stream: "face_detection"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
element_only: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Extracts face detection keypoints as a normalized landmarks.
|
||||
node {
|
||||
calculator: "DetectionToLandmarksCalculator"
|
||||
input_stream: "DETECTION:face_detection"
|
||||
output_stream: "LANDMARKS:face_landmarks"
|
||||
}
|
||||
|
||||
# Extracts the input image frame dimensions as a separate packet.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_image"
|
||||
output_stream: "SIZE:input_image_size"
|
||||
}
|
||||
|
||||
# Applies smoothing to the face landmarks previously extracted from the face
|
||||
# detection keypoints.
|
||||
node {
|
||||
calculator: "FaceLandmarksSmoothing"
|
||||
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||
input_stream: "IMAGE_SIZE:input_image_size"
|
||||
output_stream: "NORM_FILTERED_LANDMARKS:smoothed_face_landmarks"
|
||||
}
|
||||
|
||||
# Converts smoothed face landmarks back into the detection format.
|
||||
node {
|
||||
calculator: "LandmarksToDetectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:smoothed_face_landmarks"
|
||||
output_stream: "DETECTION:smoothed_face_detection"
|
||||
}
|
||||
|
||||
# Puts the smoothed single face detection back into a collection to simplify
|
||||
# passing the result into the `FaceGeometryFromDetection` subgraph.
|
||||
node {
|
||||
calculator: "ConcatenateDetectionVectorCalculator"
|
||||
input_stream: "smoothed_face_detection"
|
||||
output_stream: "multi_smoothed_face_detection"
|
||||
}
|
||||
|
||||
# Computes face geometry from the single face detection.
|
||||
node {
|
||||
calculator: "FaceGeometryFromDetection"
|
||||
input_stream: "MULTI_FACE_DETECTION:multi_smoothed_face_detection"
|
||||
input_stream: "IMAGE_SIZE:input_image_size"
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
# MediaPipe subgraph that extracts geometry from a single face using the face
|
||||
# landmark pipeline on an input GPU image. The face landmarks are also
|
||||
# "smoothed" to achieve better visual results.
|
||||
|
||||
type: "SingleFaceGeometryFromLandmarksGpu"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "IMAGE:input_image"
|
||||
|
||||
# Environment that describes the current virtual scene.
|
||||
# (face_geometry::Environment)
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
|
||||
# A list of geometry data for a single detected face. The size of this
|
||||
# collection is at most 1 because of the single-face use in this graph.
|
||||
# (std::vector<face_geometry::FaceGeometry>)
|
||||
#
|
||||
# NOTE: if no face is detected at a particular timestamp, there will not be an
|
||||
# output packet in the `MULTI_FACE_GEOMETRY` stream for this timestamp. However,
|
||||
# the MediaPipe framework will internally inform the downstream calculators of
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
|
||||
|
||||
# Creates a packet to inform the `FaceLandmarkFrontGpu` subgraph to detect at
|
||||
# most 1 face.
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:num_faces"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
|
||||
packet { int_value: 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detects faces and corresponding landmarks using the face
|
||||
# landmark pipeline.
|
||||
node {
|
||||
calculator: "FaceLandmarkFrontGpu"
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_side_packet: "NUM_FACES:num_faces"
|
||||
output_stream: "LANDMARKS:multi_face_landmarks"
|
||||
}
|
||||
|
||||
# Extracts a single set of face landmarks associated with the most prominent
|
||||
# face detected from a collection.
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListVectorCalculator"
|
||||
input_stream: "multi_face_landmarks"
|
||||
output_stream: "face_landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
element_only: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Extracts the input image frame dimensions as a separate packet.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_image"
|
||||
output_stream: "SIZE:input_image_size"
|
||||
}
|
||||
|
||||
# Applies smoothing to the single set of face landmarks.
|
||||
node {
|
||||
calculator: "FaceLandmarksSmoothing"
|
||||
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||
input_stream: "IMAGE_SIZE:input_image_size"
|
||||
output_stream: "NORM_FILTERED_LANDMARKS:smoothed_face_landmarks"
|
||||
}
|
||||
|
||||
# Puts the single set of smoothed landmarks back into a collection to simplify
|
||||
# passing the result into the `FaceGeometryFromLandmarks` subgraph.
|
||||
node {
|
||||
calculator: "ConcatenateLandmarListVectorCalculator"
|
||||
input_stream: "smoothed_face_landmarks"
|
||||
output_stream: "multi_smoothed_face_landmarks"
|
||||
}
|
||||
|
||||
# Computes face geometry from face landmarks for a single face.
|
||||
node {
|
||||
calculator: "FaceGeometryFromLandmarks"
|
||||
input_stream: "MULTI_FACE_LANDMARKS:multi_smoothed_face_landmarks"
|
||||
input_stream: "IMAGE_SIZE:input_image_size"
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
# MediaPipe subgraph that detects a single face and corresponding landmarks on
|
||||
# a input GPU image. The landmarks are also "smoothed" to achieve better visual
|
||||
# results.
|
||||
|
||||
type: "SingleFaceSmoothLandmarkGpu"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "IMAGE:input_image"
|
||||
|
||||
# Collection of detected/predicted faces, each represented as a list of face
|
||||
# landmarks. However, the size of this collection is always 1 because of the
|
||||
# single-face use in this graph. The decision to wrap the landmark list into a
|
||||
# collection was made to simplify passing the result into the `FaceGeometry`
|
||||
# subgraph. (std::vector<NormalizedLandmarkList>)
|
||||
#
|
||||
# NOTE: there will not be an output packet in the LANDMARKS stream for this
|
||||
# particular timestamp if none of faces detected. However, the MediaPipe
|
||||
# framework will internally inform the downstream calculators of the absence of
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:multi_face_smooth_landmarks"
|
||||
|
||||
# Creates a packet to inform the `FaceLandmarkFrontGpu` subgraph to detect at
|
||||
# most 1 face.
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:num_faces"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
|
||||
packet { int_value: 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detects faces and corresponding landmarks.
|
||||
node {
|
||||
calculator: "FaceLandmarkFrontGpu"
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_side_packet: "NUM_FACES:num_faces"
|
||||
output_stream: "LANDMARKS:multi_face_landmarks"
|
||||
}
|
||||
|
||||
# Extracts the detected face landmark list from a collection.
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListVectorCalculator"
|
||||
input_stream: "multi_face_landmarks"
|
||||
output_stream: "face_landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
element_only: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Extracts the input image frame dimensions as a separate packet.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_image"
|
||||
output_stream: "SIZE:input_image_size"
|
||||
}
|
||||
|
||||
# Applies smoothing to the single face landmarks.
|
||||
node {
|
||||
calculator: "LandmarksSmoothingCalculator"
|
||||
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||
input_stream: "IMAGE_SIZE:input_image_size"
|
||||
output_stream: "NORM_FILTERED_LANDMARKS:face_smooth_landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
|
||||
velocity_filter: {
|
||||
window_size: 5
|
||||
velocity_scale: 20.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Puts the single face smooth landmarks back into a collection to simplify
|
||||
# passing the result into the `FaceGeometry` subgraph.
|
||||
node {
|
||||
calculator: "ConcatenateLandmarListVectorCalculator"
|
||||
input_stream: "face_smooth_landmarks"
|
||||
output_stream: "multi_face_smooth_landmarks"
|
||||
}
|
||||
@@ -81,12 +81,11 @@ constexpr int kFaceLandmarkConnections[] = {
|
||||
class FaceLandmarksToRenderDataCalculator
|
||||
: public LandmarksToRenderDataCalculator {
|
||||
public:
|
||||
mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(FaceLandmarksToRenderDataCalculator);
|
||||
|
||||
mediapipe::Status FaceLandmarksToRenderDataCalculator::Open(
|
||||
CalculatorContext* cc) {
|
||||
absl::Status FaceLandmarksToRenderDataCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
options_ = cc->Options<mediapipe::LandmarksToRenderDataCalculatorOptions>();
|
||||
|
||||
@@ -95,7 +94,7 @@ mediapipe::Status FaceLandmarksToRenderDataCalculator::Open(
|
||||
landmark_connections_.push_back(kFaceLandmarkConnections[i * 2 + 1]);
|
||||
}
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -46,7 +46,6 @@ cc_library(
|
||||
"//mediapipe/calculators/core:merge_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_renderer_cpu",
|
||||
"//mediapipe/modules/hand_landmark:hand_landmark_tracking_cpu",
|
||||
"//mediapipe/modules/palm_detection:palm_detection_cpu",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
+9
-10
@@ -87,9 +87,9 @@ constexpr float kInitialZ = -10.0f;
|
||||
|
||||
class MatricesManagerCalculator : public CalculatorBase {
|
||||
public:
|
||||
static mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Device properties that will be preset by side packets
|
||||
@@ -137,8 +137,7 @@ class MatricesManagerCalculator : public CalculatorBase {
|
||||
|
||||
REGISTER_CALCULATOR(MatricesManagerCalculator);
|
||||
|
||||
mediapipe::Status MatricesManagerCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
absl::Status MatricesManagerCalculator::GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kAnchorsTag) &&
|
||||
cc->Inputs().HasTag(kIMUMatrixTag) &&
|
||||
cc->Inputs().HasTag(kUserRotationsTag) &&
|
||||
@@ -162,20 +161,20 @@ mediapipe::Status MatricesManagerCalculator::GetContract(
|
||||
cc->InputSidePackets().Tag(kFOVSidePacketTag).Set<float>();
|
||||
cc->InputSidePackets().Tag(kAspectRatioSidePacketTag).Set<float>();
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status MatricesManagerCalculator::Open(CalculatorContext* cc) {
|
||||
absl::Status MatricesManagerCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
// Set device properties from side packets
|
||||
vertical_fov_radians_ =
|
||||
cc->InputSidePackets().Tag(kFOVSidePacketTag).Get<float>();
|
||||
aspect_ratio_ =
|
||||
cc->InputSidePackets().Tag(kAspectRatioSidePacketTag).Get<float>();
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status MatricesManagerCalculator::Process(CalculatorContext* cc) {
|
||||
absl::Status MatricesManagerCalculator::Process(CalculatorContext* cc) {
|
||||
// Define each object's model matrices
|
||||
auto asset_matrices_gif =
|
||||
std::make_unique<mediapipe::TimedModelMatrixProtoList>();
|
||||
@@ -276,7 +275,7 @@ mediapipe::Status MatricesManagerCalculator::Process(CalculatorContext* cc) {
|
||||
.Get(cc->Outputs().GetId("MATRICES", 1))
|
||||
.Add(asset_matrices_1.release(), cc->InputTimestamp());
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// Using a specified rotation value in radians, generate a rotation matrix for
|
||||
|
||||
@@ -53,7 +53,7 @@ constexpr char kRenderDescriptorsTag[] = "RENDER_DATA";
|
||||
|
||||
class StickerManagerCalculator : public CalculatorBase {
|
||||
public:
|
||||
static mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kProtoDataString));
|
||||
RET_CHECK(cc->Outputs().HasTag(kAnchorsTag) &&
|
||||
cc->Outputs().HasTag(kUserRotationsTag) &&
|
||||
@@ -66,15 +66,15 @@ class StickerManagerCalculator : public CalculatorBase {
|
||||
cc->Outputs().Tag(kUserScalingsTag).Set<std::vector<UserScaling>>();
|
||||
cc->Outputs().Tag(kRenderDescriptorsTag).Set<std::vector<int>>();
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
absl::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
std::string sticker_proto_string =
|
||||
cc->Inputs().Tag(kProtoDataString).Get<std::string>();
|
||||
|
||||
@@ -138,11 +138,11 @@ class StickerManagerCalculator : public CalculatorBase {
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status Close(CalculatorContext* cc) override {
|
||||
return mediapipe::OkStatus();
|
||||
absl::Status Close(CalculatorContext* cc) override {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
+6
-9
@@ -71,7 +71,7 @@ class TrackedAnchorManagerCalculator : public CalculatorBase {
|
||||
std::vector<Anchor> previous_anchor_data_;
|
||||
|
||||
public:
|
||||
static mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kAnchorsTag) &&
|
||||
cc->Inputs().HasTag(kSentinelTag));
|
||||
RET_CHECK(cc->Outputs().HasTag(kAnchorsTag) &&
|
||||
@@ -91,19 +91,16 @@ class TrackedAnchorManagerCalculator : public CalculatorBase {
|
||||
cc->Outputs().Tag(kCancelTag).Set<int>();
|
||||
}
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
return mediapipe::OkStatus();
|
||||
}
|
||||
absl::Status Open(CalculatorContext* cc) override { return absl::OkStatus(); }
|
||||
|
||||
mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(TrackedAnchorManagerCalculator);
|
||||
|
||||
mediapipe::Status TrackedAnchorManagerCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
absl::Status TrackedAnchorManagerCalculator::Process(CalculatorContext* cc) {
|
||||
mediapipe::Timestamp timestamp = cc->InputTimestamp();
|
||||
const int sticker_sentinel = cc->Inputs().Tag(kSentinelTag).Get<int>();
|
||||
std::vector<Anchor> current_anchor_data =
|
||||
@@ -208,6 +205,6 @@ mediapipe::Status TrackedAnchorManagerCalculator::Process(
|
||||
.Tag(kBoxesOutputTag)
|
||||
.Add(pos_boxes.release(), cc->InputTimestamp());
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -89,7 +89,7 @@ float CalculateDepth(const NormalizedLandmark& center, float focal_length,
|
||||
// }
|
||||
class IrisToDepthCalculator : public CalculatorBase {
|
||||
public:
|
||||
static mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kIrisTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
|
||||
@@ -111,12 +111,12 @@ class IrisToDepthCalculator : public CalculatorBase {
|
||||
if (cc->Outputs().HasTag(kRightIrisDepthTag)) {
|
||||
cc->Outputs().Tag(kRightIrisDepthTag).Set<float>();
|
||||
}
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
|
||||
mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
float focal_length_pixels_ = -1.f;
|
||||
@@ -134,7 +134,7 @@ class IrisToDepthCalculator : public CalculatorBase {
|
||||
};
|
||||
REGISTER_CALCULATOR(IrisToDepthCalculator);
|
||||
|
||||
mediapipe::Status IrisToDepthCalculator::Open(CalculatorContext* cc) {
|
||||
absl::Status IrisToDepthCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
if (cc->InputSidePackets().HasTag(kFocalLengthPixelTag)) {
|
||||
#if defined(__APPLE__)
|
||||
@@ -155,13 +155,13 @@ mediapipe::Status IrisToDepthCalculator::Open(CalculatorContext* cc) {
|
||||
}
|
||||
|
||||
options_ = cc->Options<::mediapipe::IrisToDepthCalculatorOptions>();
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status IrisToDepthCalculator::Process(CalculatorContext* cc) {
|
||||
absl::Status IrisToDepthCalculator::Process(CalculatorContext* cc) {
|
||||
// Only process if there's input landmarks.
|
||||
if (cc->Inputs().Tag(kIrisTag).IsEmpty()) {
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& iris_landmarks =
|
||||
@@ -220,7 +220,7 @@ mediapipe::Status IrisToDepthCalculator::Process(CalculatorContext* cc) {
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
}
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void IrisToDepthCalculator::GetLeftIris(const NormalizedLandmarkList& lds,
|
||||
|
||||
@@ -108,7 +108,7 @@ float CalculateDepth(const NormalizedLandmark& center, float focal_length,
|
||||
// }
|
||||
class IrisToRenderDataCalculator : public CalculatorBase {
|
||||
public:
|
||||
static mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kIrisTag).Set<NormalizedLandmarkList>();
|
||||
cc->Outputs().Tag(kRenderDataTag).Set<RenderData>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
@@ -119,12 +119,12 @@ class IrisToRenderDataCalculator : public CalculatorBase {
|
||||
if (cc->Inputs().HasTag(kRightIrisDepthTag)) {
|
||||
cc->Inputs().Tag(kRightIrisDepthTag).Set<float>();
|
||||
}
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
|
||||
mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
void RenderIris(const NormalizedLandmarkList& iris_landmarks,
|
||||
@@ -150,15 +150,15 @@ class IrisToRenderDataCalculator : public CalculatorBase {
|
||||
};
|
||||
REGISTER_CALCULATOR(IrisToRenderDataCalculator);
|
||||
|
||||
mediapipe::Status IrisToRenderDataCalculator::Open(CalculatorContext* cc) {
|
||||
absl::Status IrisToRenderDataCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status IrisToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
absl::Status IrisToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
// Only process if there's input landmarks.
|
||||
if (cc->Inputs().Tag(kIrisTag).IsEmpty()) {
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::IrisToRenderDataCalculatorOptions>();
|
||||
@@ -212,7 +212,7 @@ mediapipe::Status IrisToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
cc->Outputs()
|
||||
.Tag(kRenderDataTag)
|
||||
.Add(render_data.release(), cc->InputTimestamp());
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void IrisToRenderDataCalculator::AddTextRenderData(
|
||||
|
||||
@@ -215,28 +215,27 @@ constexpr int kEyeLandmarkIndicesInFaceLandmarks[] = {
|
||||
//
|
||||
class UpdateFaceLandmarksCalculator : public CalculatorBase {
|
||||
public:
|
||||
static mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kFaceLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kNewEyeLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
|
||||
cc->Outputs().Tag(kUpdatedFaceLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
mediapipe::Status Open(CalculatorContext* cc) {
|
||||
absl::Status Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(UpdateFaceLandmarksCalculator);
|
||||
|
||||
mediapipe::Status UpdateFaceLandmarksCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
absl::Status UpdateFaceLandmarksCalculator::Process(CalculatorContext* cc) {
|
||||
if (cc->Inputs().Tag(kFaceLandmarksTag).IsEmpty() ||
|
||||
cc->Inputs().Tag(kNewEyeLandmarksTag).IsEmpty()) {
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
const auto& face_landmarks =
|
||||
cc->Inputs().Tag(kFaceLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
@@ -263,7 +262,7 @@ mediapipe::Status UpdateFaceLandmarksCalculator::Process(
|
||||
.Tag(kUpdatedFaceLandmarksTag)
|
||||
.Add(refined_face_landmarks.release(), cc->InputTimestamp());
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -56,8 +56,6 @@ cc_library(
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:constant_side_packet_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_model_calculator",
|
||||
"//mediapipe/calculators/util:local_file_contents_calculator",
|
||||
"//mediapipe/calculators/video:opencv_video_decoder_calculator",
|
||||
"//mediapipe/calculators/video:opencv_video_encoder_calculator",
|
||||
"//mediapipe/graphs/object_detection_3d/subgraphs:renderer_cpu",
|
||||
|
||||
+36
-43
@@ -37,6 +37,7 @@ namespace {
|
||||
constexpr char kAnnotationTag[] = "ANNOTATIONS";
|
||||
constexpr char kModelMatricesTag[] = "MODEL_MATRICES";
|
||||
|
||||
using Matrix3fRM = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>;
|
||||
using Matrix4fRM = Eigen::Matrix<float, 4, 4, Eigen::RowMajor>;
|
||||
|
||||
} // namespace
|
||||
@@ -66,14 +67,14 @@ class AnnotationsToModelMatricesCalculator : public CalculatorBase {
|
||||
AnnotationsToModelMatricesCalculator& operator=(
|
||||
const AnnotationsToModelMatricesCalculator&) = delete;
|
||||
|
||||
static mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
|
||||
mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
mediapipe::Status GetModelMatricesForAnnotations(
|
||||
absl::Status GetModelMatricesForAnnotations(
|
||||
const FrameAnnotation& annotations,
|
||||
TimedModelMatrixProtoList* model_matrix_list);
|
||||
|
||||
@@ -83,7 +84,7 @@ class AnnotationsToModelMatricesCalculator : public CalculatorBase {
|
||||
};
|
||||
REGISTER_CALCULATOR(AnnotationsToModelMatricesCalculator);
|
||||
|
||||
mediapipe::Status AnnotationsToModelMatricesCalculator::GetContract(
|
||||
absl::Status AnnotationsToModelMatricesCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kAnnotationTag)) << "No input stream found.";
|
||||
if (cc->Inputs().HasTag(kAnnotationTag)) {
|
||||
@@ -101,11 +102,10 @@ mediapipe::Status AnnotationsToModelMatricesCalculator::GetContract(
|
||||
if (cc->InputSidePackets().HasTag("MODEL_TRANSFORMATION")) {
|
||||
cc->InputSidePackets().Tag("MODEL_TRANSFORMATION").Set<float[]>();
|
||||
}
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status AnnotationsToModelMatricesCalculator::Open(
|
||||
CalculatorContext* cc) {
|
||||
absl::Status AnnotationsToModelMatricesCalculator::Open(CalculatorContext* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kAnnotationTag));
|
||||
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
@@ -131,10 +131,10 @@ mediapipe::Status AnnotationsToModelMatricesCalculator::Open(
|
||||
model_transformation_.setIdentity();
|
||||
}
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status AnnotationsToModelMatricesCalculator::Process(
|
||||
absl::Status AnnotationsToModelMatricesCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
auto model_matrices = std::make_unique<TimedModelMatrixProtoList>();
|
||||
|
||||
@@ -142,73 +142,66 @@ mediapipe::Status AnnotationsToModelMatricesCalculator::Process(
|
||||
cc->Inputs().Tag(kAnnotationTag).Get<FrameAnnotation>();
|
||||
|
||||
if (!GetModelMatricesForAnnotations(annotations, model_matrices.get()).ok()) {
|
||||
return mediapipe::InvalidArgumentError("Error in GetModelMatricesForBoxes");
|
||||
return absl::InvalidArgumentError("Error in GetModelMatricesForBoxes");
|
||||
}
|
||||
cc->Outputs()
|
||||
.Tag(kModelMatricesTag)
|
||||
.Add(model_matrices.release(), cc->InputTimestamp());
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status
|
||||
absl::Status
|
||||
AnnotationsToModelMatricesCalculator::GetModelMatricesForAnnotations(
|
||||
const FrameAnnotation& annotations,
|
||||
TimedModelMatrixProtoList* model_matrix_list) {
|
||||
if (model_matrix_list == nullptr) {
|
||||
return mediapipe::InvalidArgumentError("model_matrix_list is nullptr");
|
||||
return absl::InvalidArgumentError("model_matrix_list is nullptr");
|
||||
}
|
||||
model_matrix_list->clear_model_matrix();
|
||||
|
||||
Box box("category");
|
||||
for (const auto& object : annotations.annotations()) {
|
||||
TimedModelMatrixProto* model_matrix = model_matrix_list->add_model_matrix();
|
||||
model_matrix->set_id(object.object_id());
|
||||
|
||||
// Fit a box to the original vertices to estimate the scale of the box
|
||||
std::vector<Eigen::Vector3f> vertices;
|
||||
for (const auto& keypoint : object.keypoints()) {
|
||||
const auto& point = keypoint.point_3d();
|
||||
Eigen::Vector3f p(point.x(), point.y(), point.z());
|
||||
vertices.emplace_back(p);
|
||||
}
|
||||
box.Fit(vertices);
|
||||
// Get object rotation, translation and scale.
|
||||
const auto object_rotation =
|
||||
Eigen::Map<const Matrix3fRM>(object.rotation().data());
|
||||
const auto object_translation =
|
||||
Eigen::Map<const Eigen::Vector3f>(object.translation().data());
|
||||
const auto object_scale =
|
||||
Eigen::Map<const Eigen::Vector3f>(object.scale().data());
|
||||
|
||||
// Re-scale the box if necessary
|
||||
Eigen::Vector3f estimated_scale = box.GetScale();
|
||||
vertices.clear();
|
||||
for (const auto& keypoint : object.keypoints()) {
|
||||
const auto& point = keypoint.point_3d();
|
||||
Eigen::Vector3f p(point.x(), point.y(), point.z());
|
||||
vertices.emplace_back(p);
|
||||
}
|
||||
box.Fit(vertices);
|
||||
// Compose object transformation matrix.
|
||||
Matrix4fRM object_transformation;
|
||||
object_transformation.setIdentity();
|
||||
object_transformation.topLeftCorner<3, 3>() = object_rotation;
|
||||
object_transformation.topRightCorner<3, 1>() = object_translation;
|
||||
|
||||
Matrix4fRM object_transformation = box.GetTransformation();
|
||||
Matrix4fRM model_view;
|
||||
Matrix4fRM pursuit_model;
|
||||
Matrix4fRM objectron_model;
|
||||
// The reference view is
|
||||
//
|
||||
// ref << 0., 0., 1., 0.,
|
||||
// -1., 0., 0., 0.,
|
||||
// 0., -1., 0., 0.,
|
||||
// 0., 0., 0., 1.;
|
||||
// We have pursuit_model * model = model_view, to get pursuit_model:
|
||||
// pursuit_model = model_view * model^-1
|
||||
// We have objectron_model * model = model_view, to get objectron_model:
|
||||
// objectron_model = model_view * model^-1
|
||||
// clang-format off
|
||||
pursuit_model << 0.0, 1.0, 0.0, 0.0,
|
||||
1.0, 0.0, 0.0, 0.0,
|
||||
0.0, 0.0, 1.0, 0.0,
|
||||
0.0, 0.0, 0.0, 1.0;
|
||||
objectron_model << 1.0, 0.0, 0.0, 0.0,
|
||||
0.0, -1., 0.0, 0.0,
|
||||
0.0, 0.0, 1.0, 0.0,
|
||||
0.0, 0.0, 0.0, 1.0;
|
||||
// clang-format on
|
||||
|
||||
// Re-scale the CAD model to the scale of the estimated bounding box.
|
||||
const Eigen::Vector3f scale = model_scale_.cwiseProduct(estimated_scale);
|
||||
const Eigen::Vector3f scale = model_scale_.cwiseProduct(object_scale);
|
||||
const Matrix4fRM model =
|
||||
model_transformation_.array().colwise() * scale.homogeneous().array();
|
||||
|
||||
// Finally compute the model_view matrix.
|
||||
model_view = pursuit_model * object_transformation * model;
|
||||
model_view = objectron_model * object_transformation * model;
|
||||
|
||||
for (int i = 0; i < model_view.rows(); ++i) {
|
||||
for (int j = 0; j < model_view.cols(); ++j) {
|
||||
@@ -216,7 +209,7 @@ AnnotationsToModelMatricesCalculator::GetModelMatricesForAnnotations(
|
||||
}
|
||||
}
|
||||
}
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
+9
-11
@@ -98,11 +98,11 @@ class AnnotationsToRenderDataCalculator : public CalculatorBase {
|
||||
AnnotationsToRenderDataCalculator& operator=(
|
||||
const AnnotationsToRenderDataCalculator&) = delete;
|
||||
|
||||
static mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
|
||||
mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
static void SetRenderAnnotationColorThickness(
|
||||
@@ -134,7 +134,7 @@ class AnnotationsToRenderDataCalculator : public CalculatorBase {
|
||||
};
|
||||
REGISTER_CALCULATOR(AnnotationsToRenderDataCalculator);
|
||||
|
||||
mediapipe::Status AnnotationsToRenderDataCalculator::GetContract(
|
||||
absl::Status AnnotationsToRenderDataCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kAnnotationTag)) << "No input stream found.";
|
||||
if (cc->Inputs().HasTag(kAnnotationTag)) {
|
||||
@@ -142,19 +142,17 @@ mediapipe::Status AnnotationsToRenderDataCalculator::GetContract(
|
||||
}
|
||||
cc->Outputs().Tag(kRenderDataTag).Set<RenderData>();
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status AnnotationsToRenderDataCalculator::Open(
|
||||
CalculatorContext* cc) {
|
||||
absl::Status AnnotationsToRenderDataCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
options_ = cc->Options<AnnotationsToRenderDataCalculatorOptions>();
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status AnnotationsToRenderDataCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
absl::Status AnnotationsToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
auto render_data = absl::make_unique<RenderData>();
|
||||
bool visualize_depth = options_.visualize_landmark_depth();
|
||||
float z_min = 0.f;
|
||||
@@ -215,7 +213,7 @@ mediapipe::Status AnnotationsToRenderDataCalculator::Process(
|
||||
.Tag(kRenderDataTag)
|
||||
.Add(render_data.release(), cc->InputTimestamp());
|
||||
|
||||
return mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void AnnotationsToRenderDataCalculator::AddConnectionToRenderData(
|
||||
|
||||
+26
-28
@@ -128,10 +128,10 @@ class GlAnimationOverlayCalculator : public CalculatorBase {
|
||||
GlAnimationOverlayCalculator() {}
|
||||
~GlAnimationOverlayCalculator();
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract *cc);
|
||||
static absl::Status GetContract(CalculatorContract *cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext *cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext *cc) override;
|
||||
absl::Status Open(CalculatorContext *cc) override;
|
||||
absl::Status Process(CalculatorContext *cc) override;
|
||||
|
||||
private:
|
||||
bool has_video_stream_ = false;
|
||||
@@ -171,11 +171,11 @@ class GlAnimationOverlayCalculator : public CalculatorBase {
|
||||
float *vertical_fov_degrees);
|
||||
|
||||
int GetAnimationFrameIndex(Timestamp timestamp);
|
||||
::mediapipe::Status GlSetup();
|
||||
::mediapipe::Status GlBind(const TriangleMesh &triangle_mesh,
|
||||
const GlTexture &texture);
|
||||
::mediapipe::Status GlRender(const TriangleMesh &triangle_mesh,
|
||||
const float *model_matrix);
|
||||
absl::Status GlSetup();
|
||||
absl::Status GlBind(const TriangleMesh &triangle_mesh,
|
||||
const GlTexture &texture);
|
||||
absl::Status GlRender(const TriangleMesh &triangle_mesh,
|
||||
const float *model_matrix);
|
||||
void InitializePerspectiveMatrix(float aspect_ratio,
|
||||
float vertical_fov_degrees, float z_near,
|
||||
float z_far);
|
||||
@@ -198,8 +198,7 @@ class GlAnimationOverlayCalculator : public CalculatorBase {
|
||||
REGISTER_CALCULATOR(GlAnimationOverlayCalculator);
|
||||
|
||||
// static
|
||||
::mediapipe::Status GlAnimationOverlayCalculator::GetContract(
|
||||
CalculatorContract *cc) {
|
||||
absl::Status GlAnimationOverlayCalculator::GetContract(CalculatorContract *cc) {
|
||||
MP_RETURN_IF_ERROR(
|
||||
GlCalculatorHelper::SetupInputSidePackets(&(cc->InputSidePackets())));
|
||||
if (cc->Inputs().HasTag("VIDEO")) {
|
||||
@@ -236,7 +235,7 @@ REGISTER_CALCULATOR(GlAnimationOverlayCalculator);
|
||||
cc->InputSidePackets().Tag("MASK_ASSET").Set<std::string>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void GlAnimationOverlayCalculator::CalculateTriangleMeshNormals(
|
||||
@@ -515,7 +514,7 @@ void GlAnimationOverlayCalculator::ComputeAspectRatioAndFovFromCameraParameters(
|
||||
std::atan(camera_parameters.portrait_height() * 0.5f) * 2 * 180 / M_PI;
|
||||
}
|
||||
|
||||
::mediapipe::Status GlAnimationOverlayCalculator::Open(CalculatorContext *cc) {
|
||||
absl::Status GlAnimationOverlayCalculator::Open(CalculatorContext *cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
MP_RETURN_IF_ERROR(helper_.Open(cc));
|
||||
|
||||
@@ -562,7 +561,7 @@ void GlAnimationOverlayCalculator::ComputeAspectRatioAndFovFromCameraParameters(
|
||||
loaded_animation = LoadAnimationAndroid(mask_asset_name, &mask_meshes_);
|
||||
if (!loaded_animation) {
|
||||
LOG(ERROR) << "Failed to load mask asset.";
|
||||
return ::mediapipe::UnknownError("Failed to load mask asset.");
|
||||
return absl::UnknownError("Failed to load mask asset.");
|
||||
}
|
||||
}
|
||||
loaded_animation = LoadAnimationAndroid(asset_name, &triangle_meshes_);
|
||||
@@ -571,10 +570,10 @@ void GlAnimationOverlayCalculator::ComputeAspectRatioAndFovFromCameraParameters(
|
||||
#endif
|
||||
if (!loaded_animation) {
|
||||
LOG(ERROR) << "Failed to load animation asset.";
|
||||
return ::mediapipe::UnknownError("Failed to load animation asset.");
|
||||
return absl::UnknownError("Failed to load animation asset.");
|
||||
}
|
||||
|
||||
return helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
|
||||
return helper_.RunInGlContext([this, &cc]() -> absl::Status {
|
||||
if (cc->InputSidePackets().HasTag("MASK_TEXTURE")) {
|
||||
const auto &mask_texture =
|
||||
cc->InputSidePackets().Tag("MASK_TEXTURE").Get<AssetTextureFormat>();
|
||||
@@ -591,7 +590,7 @@ void GlAnimationOverlayCalculator::ComputeAspectRatioAndFovFromCameraParameters(
|
||||
VLOG(2) << "Input texture size: " << texture_.width() << ", "
|
||||
<< texture_.height() << std::endl;
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -624,9 +623,8 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
}
|
||||
}
|
||||
|
||||
::mediapipe::Status GlAnimationOverlayCalculator::Process(
|
||||
CalculatorContext *cc) {
|
||||
return helper_.RunInGlContext([this, &cc]() -> mediapipe::Status {
|
||||
absl::Status GlAnimationOverlayCalculator::Process(CalculatorContext *cc) {
|
||||
return helper_.RunInGlContext([this, &cc]() -> absl::Status {
|
||||
if (!initialized_) {
|
||||
MP_RETURN_IF_ERROR(GlSetup());
|
||||
initialized_ = true;
|
||||
@@ -663,7 +661,7 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
if (has_video_stream_ && !(cc->Inputs().Tag("VIDEO").IsEmpty())) {
|
||||
auto result = cc->Inputs().Tag("VIDEO").Value().Consume<GpuBuffer>();
|
||||
if (result.ok()) {
|
||||
input_frame = std::move(result).ValueOrDie();
|
||||
input_frame = std::move(result).value();
|
||||
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
input_frame->GetGlTextureBufferSharedPtr()->Reuse();
|
||||
#endif
|
||||
@@ -679,7 +677,7 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
dst = helper_.CreateDestinationTexture(width, height);
|
||||
} else {
|
||||
// We have an input video stream, but not for this frame. Don't render!
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
helper_.BindFramebuffer(dst);
|
||||
|
||||
@@ -759,11 +757,11 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
TagOrIndex(&(cc->Outputs()), "OUTPUT", 0)
|
||||
.Add(output.release(), cc->InputTimestamp());
|
||||
GLCHECK(glFrontFace(GL_CCW));
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
});
|
||||
}
|
||||
|
||||
::mediapipe::Status GlAnimationOverlayCalculator::GlSetup() {
|
||||
absl::Status GlAnimationOverlayCalculator::GlSetup() {
|
||||
// Load vertex and fragment shaders
|
||||
const GLint attr_location[NUM_ATTRIBUTES] = {
|
||||
ATTRIB_VERTEX,
|
||||
@@ -881,10 +879,10 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
GLCHECK(glGetUniformLocation(program_, "perspectiveMatrix"));
|
||||
model_matrix_uniform_ =
|
||||
GLCHECK(glGetUniformLocation(program_, "modelMatrix"));
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status GlAnimationOverlayCalculator::GlBind(
|
||||
absl::Status GlAnimationOverlayCalculator::GlBind(
|
||||
const TriangleMesh &triangle_mesh, const GlTexture &texture) {
|
||||
GLCHECK(glUseProgram(program_));
|
||||
|
||||
@@ -915,16 +913,16 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
|
||||
GLCHECK(glUniformMatrix4fv(perspective_matrix_uniform_, 1, GL_FALSE,
|
||||
perspective_matrix_));
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status GlAnimationOverlayCalculator::GlRender(
|
||||
absl::Status GlAnimationOverlayCalculator::GlRender(
|
||||
const TriangleMesh &triangle_mesh, const float *model_matrix) {
|
||||
GLCHECK(glUniformMatrix4fv(model_matrix_uniform_, 1, GL_FALSE, model_matrix));
|
||||
GLCHECK(glDrawElements(GL_TRIANGLES, triangle_mesh.index_count,
|
||||
GL_UNSIGNED_SHORT,
|
||||
triangle_mesh.triangle_indices.get()));
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
GlAnimationOverlayCalculator::~GlAnimationOverlayCalculator() {
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# Copyright 2021 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
java_library(
|
||||
name = "obj_parser_lib",
|
||||
srcs = [
|
||||
"ObjParserMain.java",
|
||||
"SimpleObjParser.java",
|
||||
],
|
||||
javacopts = ["-Xep:DefaultPackage:OFF"],
|
||||
)
|
||||
|
||||
java_binary(
|
||||
name = "ObjParser",
|
||||
javacopts = ["-Xep:DefaultPackage:OFF"],
|
||||
main_class = "ObjParserMain",
|
||||
runtime_deps = [
|
||||
":obj_parser_lib",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,205 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileFilter;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.PrintWriter;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* Class for running desktop-side parsing/packing routines on .obj AR assets. Usage: ObjParser
|
||||
* --input_dir=[INPUT_DIRECTORY] --output_dir=[OUTPUT_DIRECTORY] where INPUT_DIRECTORY is the folder
|
||||
* with asset obj files to process, and OUTPUT_DIRECTORY is the folder where processed asset uuu
|
||||
* file should be placed.
|
||||
*
|
||||
* <p>NOTE: Directories are assumed to be absolute paths.
|
||||
*/
|
||||
public final class ObjParserMain {
|
||||
// Simple FileFilter implementation to let us walk over only our .obj files in a particular
|
||||
// directory.
|
||||
private static final class ObjFileFilter implements FileFilter {
|
||||
ObjFileFilter() {
|
||||
// Nothing to do here.
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accept(File file) {
|
||||
return file.getName().endsWith(".obj");
|
||||
}
|
||||
}
|
||||
|
||||
// File extension for binary output files; tagged onto end of initial file extension.
|
||||
private static final String BINARY_FILE_EXT = ".uuu";
|
||||
private static final String INPUT_DIR_FLAG = "--input_dir=";
|
||||
private static final String OUTPUT_DIR_FLAG = "--output_dir=";
|
||||
private static final float DEFAULT_VERTEX_SCALE_FACTOR = 30.0f;
|
||||
private static final double NS_TO_SECONDS = 1e9;
|
||||
|
||||
public final PrintWriter writer;
|
||||
|
||||
public ObjParserMain() {
|
||||
super();
|
||||
this.writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(System.out, UTF_8)));
|
||||
}
|
||||
|
||||
// Simple overridable logging function.
|
||||
protected void logString(String infoLog) {
|
||||
writer.println(infoLog);
|
||||
}
|
||||
|
||||
/*
|
||||
* Main program logic: parse command-line arguments and perform actions.
|
||||
*/
|
||||
public void run(String inDirectory, String outDirectory) {
|
||||
if (inDirectory.isEmpty()) {
|
||||
logString("Error: Must provide input directory with " + INPUT_DIR_FLAG);
|
||||
return;
|
||||
}
|
||||
if (outDirectory.isEmpty()) {
|
||||
logString("Error: Must provide output directory with " + OUTPUT_DIR_FLAG);
|
||||
return;
|
||||
}
|
||||
|
||||
File dirAsFile = new File(inDirectory);
|
||||
ObjFileFilter objFileFilter = new ObjFileFilter();
|
||||
File[] objFiles = dirAsFile.listFiles(objFileFilter);
|
||||
|
||||
FileOutputStream outputStream = null;
|
||||
logString("Parsing directory: " + inDirectory);
|
||||
// We need frames processed in correct order.
|
||||
Arrays.sort(objFiles);
|
||||
|
||||
for (File objFile : objFiles) {
|
||||
String fileName = objFile.getAbsolutePath();
|
||||
|
||||
// Just take the file name of the first processed frame.
|
||||
if (outputStream == null) {
|
||||
String outputFileName = outDirectory + objFile.getName() + BINARY_FILE_EXT;
|
||||
try {
|
||||
// Create new file here, if we can.
|
||||
outputStream = new FileOutputStream(outputFileName);
|
||||
logString("Created outfile: " + outputFileName);
|
||||
} catch (Exception e) {
|
||||
logString("Error creating outfile: " + e.toString());
|
||||
e.printStackTrace(writer);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Process each file into the stream.
|
||||
logString("Processing file: " + fileName);
|
||||
processFile(fileName, outputStream);
|
||||
}
|
||||
|
||||
// Finally close the stream out.
|
||||
try {
|
||||
if (outputStream != null) {
|
||||
outputStream.close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logString("Error trying to close output stream: " + e.toString());
|
||||
e.printStackTrace(writer);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Entrypoint for command-line executable.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
// Parse flags
|
||||
String inDirectory = "";
|
||||
String outDirectory = "";
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
if (args[i].startsWith(INPUT_DIR_FLAG)) {
|
||||
inDirectory = args[i].substring(INPUT_DIR_FLAG.length());
|
||||
// Make sure this will be treated as a directory
|
||||
if (!inDirectory.endsWith("/")) {
|
||||
inDirectory += "/";
|
||||
}
|
||||
}
|
||||
if (args[i].startsWith(OUTPUT_DIR_FLAG)) {
|
||||
outDirectory = args[i].substring(OUTPUT_DIR_FLAG.length());
|
||||
// Make sure this will be treated as a directory
|
||||
if (!outDirectory.endsWith("/")) {
|
||||
outDirectory += "/";
|
||||
}
|
||||
}
|
||||
}
|
||||
ObjParserMain parser = new ObjParserMain();
|
||||
parser.run(inDirectory, outDirectory);
|
||||
parser.writer.flush();
|
||||
}
|
||||
|
||||
/*
|
||||
* Internal helper function to parse a .obj from an infile name and stream the resulting data
|
||||
* directly out in binary-dump format to outputStream.
|
||||
*/
|
||||
private void processFile(String infileName, OutputStream outputStream) {
|
||||
long start = System.nanoTime();
|
||||
|
||||
// First we parse the obj.
|
||||
SimpleObjParser objParser = new SimpleObjParser(infileName, DEFAULT_VERTEX_SCALE_FACTOR);
|
||||
if (!objParser.parse()) {
|
||||
logString("Error parsing .obj file before processing");
|
||||
return;
|
||||
}
|
||||
|
||||
final float[] vertices = objParser.getVertices();
|
||||
final float[] textureCoords = objParser.getTextureCoords();
|
||||
final ArrayList<Short> triangleList = objParser.getTriangles();
|
||||
|
||||
// Overall byte count to stream: 12 for the 3 list-length ints, and then 4 for each vertex and
|
||||
// texCoord int, and finally 2 for each triangle index short.
|
||||
final int bbSize =
|
||||
12 + 4 * vertices.length + 4 * textureCoords.length + 2 * triangleList.size();
|
||||
|
||||
// Ensure ByteBuffer is native order, just like we want to read it in, but is NOT direct, so
|
||||
// we can call .array() on it.
|
||||
ByteBuffer bb = ByteBuffer.allocate(bbSize);
|
||||
bb.order(ByteOrder.nativeOrder());
|
||||
|
||||
bb.putInt(vertices.length);
|
||||
bb.putInt(textureCoords.length);
|
||||
bb.putInt(triangleList.size());
|
||||
logString(String.format("Writing... Vertices: %d, TextureCoords: %d, Indices: %d.%n",
|
||||
vertices.length, textureCoords.length, triangleList.size()));
|
||||
for (float vertex : vertices) {
|
||||
bb.putFloat(vertex);
|
||||
}
|
||||
for (float textureCoord : textureCoords) {
|
||||
bb.putFloat(textureCoord);
|
||||
}
|
||||
for (Short vertexIndex : triangleList) {
|
||||
bb.putShort(vertexIndex.shortValue());
|
||||
}
|
||||
bb.position(0);
|
||||
try {
|
||||
outputStream.write(bb.array(), 0, bbSize);
|
||||
logString(String.format("Processing successful! Took %.4f seconds.%n",
|
||||
(System.nanoTime() - start) / NS_TO_SECONDS));
|
||||
} catch (Exception e) {
|
||||
logString("Error writing during processing: " + e.toString());
|
||||
e.printStackTrace(writer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Class for parsing a single .obj file into openGL-usable pieces.
|
||||
*
|
||||
* <p>Usage:
|
||||
*
|
||||
* <p>SimpleObjParser objParser = new SimpleObjParser("animations/cow/cow320.obj", .015f);
|
||||
*
|
||||
* <p>if (objParser.parse()) { ... }
|
||||
*/
|
||||
public class SimpleObjParser {
|
||||
private static class ShortPair {
|
||||
private final Short first;
|
||||
private final Short second;
|
||||
|
||||
public ShortPair(Short newFirst, Short newSecond) {
|
||||
first = newFirst;
|
||||
second = newSecond;
|
||||
}
|
||||
|
||||
public Short getFirst() {
|
||||
return first;
|
||||
}
|
||||
|
||||
public Short getSecond() {
|
||||
return second;
|
||||
}
|
||||
}
|
||||
|
||||
private static final String TAG = SimpleObjParser.class.getSimpleName();
|
||||
private static final boolean DEBUG = false;
|
||||
private static final int INVALID_INDEX = -1;
|
||||
private static final int POSITIONS_COORDS_PER_VERTEX = 3;
|
||||
private static final int TEXTURE_COORDS_PER_VERTEX = 2;
|
||||
private final String fileName;
|
||||
|
||||
// Since .obj doesn't tie together texture coordinates and vertex
|
||||
// coordinates, but OpenGL does, we need to keep a map of all such pairings that occur in
|
||||
// our face list.
|
||||
private final HashMap<ShortPair, Short> vertexTexCoordMap;
|
||||
|
||||
// Internal (de-coupled) unique vertices and texture coordinates
|
||||
private ArrayList<Float> vertices;
|
||||
private ArrayList<Float> textureCoords;
|
||||
|
||||
// Data we expose to openGL for rendering
|
||||
private float[] finalizedVertices;
|
||||
private float[] finalizedTextureCoords;
|
||||
private ArrayList<Short> finalizedTriangles;
|
||||
|
||||
// So we only display warnings about dropped w-coordinates once
|
||||
private boolean vertexCoordIgnoredWarning;
|
||||
private boolean textureCoordIgnoredWarning;
|
||||
private boolean startedProcessingFaces;
|
||||
|
||||
private int numPrimitiveVertices;
|
||||
private int numPrimitiveTextureCoords;
|
||||
private int numPrimitiveFaces;
|
||||
|
||||
// For scratchwork, so we don't have to keep reallocating
|
||||
private float[] tempCoords;
|
||||
|
||||
// We scale all our position coordinates uniformly by this factor
|
||||
private float objectUniformScaleFactor;
|
||||
|
||||
public SimpleObjParser(String objFile, float scaleFactor) {
|
||||
objectUniformScaleFactor = scaleFactor;
|
||||
|
||||
fileName = objFile;
|
||||
vertices = new ArrayList<Float>();
|
||||
textureCoords = new ArrayList<Float>();
|
||||
|
||||
vertexTexCoordMap = new HashMap<ShortPair, Short>();
|
||||
finalizedTriangles = new ArrayList<Short>();
|
||||
|
||||
tempCoords = new float[Math.max(POSITIONS_COORDS_PER_VERTEX, TEXTURE_COORDS_PER_VERTEX)];
|
||||
numPrimitiveFaces = 0;
|
||||
|
||||
vertexCoordIgnoredWarning = false;
|
||||
textureCoordIgnoredWarning = false;
|
||||
startedProcessingFaces = false;
|
||||
}
|
||||
|
||||
// Simple helper wrapper function
|
||||
private void debugLogString(String message) {
|
||||
if (DEBUG) {
|
||||
System.out.println(message);
|
||||
}
|
||||
}
|
||||
|
||||
private void parseVertex(String[] linePieces) {
|
||||
// Note: Traditionally xyzw is acceptable as a format, with w defaulting to 1.0, but for now
|
||||
// we only parse xyz.
|
||||
if (linePieces.length < POSITIONS_COORDS_PER_VERTEX + 1
|
||||
|| linePieces.length > POSITIONS_COORDS_PER_VERTEX + 2) {
|
||||
System.out.println("Malformed vertex coordinate specification, assuming xyz format only.");
|
||||
return;
|
||||
} else if (linePieces.length == POSITIONS_COORDS_PER_VERTEX + 2 && !vertexCoordIgnoredWarning) {
|
||||
System.out.println(
|
||||
"Only x, y, and z parsed for vertex coordinates; w coordinates will be ignored.");
|
||||
vertexCoordIgnoredWarning = true;
|
||||
}
|
||||
|
||||
boolean success = true;
|
||||
try {
|
||||
for (int i = 1; i < POSITIONS_COORDS_PER_VERTEX + 1; i++) {
|
||||
tempCoords[i - 1] = Float.parseFloat(linePieces[i]);
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
success = false;
|
||||
System.out.println("Malformed vertex coordinate error: " + e.toString());
|
||||
}
|
||||
|
||||
if (success) {
|
||||
for (int i = 0; i < POSITIONS_COORDS_PER_VERTEX; i++) {
|
||||
vertices.add(Float.valueOf(tempCoords[i] * objectUniformScaleFactor));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void parseTextureCoordinate(String[] linePieces) {
|
||||
// Similar to vertices, uvw is acceptable as a format, with w defaulting to 0.0, but for now we
|
||||
// only parse uv.
|
||||
if (linePieces.length < TEXTURE_COORDS_PER_VERTEX + 1
|
||||
|| linePieces.length > TEXTURE_COORDS_PER_VERTEX + 2) {
|
||||
System.out.println("Malformed texture coordinate specification, assuming uv format only.");
|
||||
return;
|
||||
} else if (linePieces.length == (TEXTURE_COORDS_PER_VERTEX + 2)
|
||||
&& !textureCoordIgnoredWarning) {
|
||||
debugLogString("Only u and v parsed for texture coordinates; w coordinates will be ignored.");
|
||||
textureCoordIgnoredWarning = true;
|
||||
}
|
||||
|
||||
boolean success = true;
|
||||
try {
|
||||
for (int i = 1; i < TEXTURE_COORDS_PER_VERTEX + 1; i++) {
|
||||
tempCoords[i - 1] = Float.parseFloat(linePieces[i]);
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
success = false;
|
||||
System.out.println("Malformed texture coordinate error: " + e.toString());
|
||||
}
|
||||
|
||||
if (success) {
|
||||
// .obj files treat (0,0) as top-left, compared to bottom-left for openGL. So invert "v"
|
||||
// texture coordinate only here.
|
||||
textureCoords.add(Float.valueOf(tempCoords[0]));
|
||||
textureCoords.add(Float.valueOf(1.0f - tempCoords[1]));
|
||||
}
|
||||
}
|
||||
|
||||
// Will return INVALID_INDEX if error occurs, and otherwise will return finalized (combined)
|
||||
// index, adding and hashing new combinations as it sees them.
|
||||
private short parseAndProcessCombinedVertexCoord(String coordString) {
|
||||
String[] coords = coordString.split("/");
|
||||
try {
|
||||
// Parse vertex and texture indices; 1-indexed from front if positive and from end of list if
|
||||
// negative.
|
||||
short vertexIndex = Short.parseShort(coords[0]);
|
||||
short textureIndex = Short.parseShort(coords[1]);
|
||||
if (vertexIndex > 0) {
|
||||
vertexIndex--;
|
||||
} else {
|
||||
vertexIndex = (short) (vertexIndex + numPrimitiveVertices);
|
||||
}
|
||||
if (textureIndex > 0) {
|
||||
textureIndex--;
|
||||
} else {
|
||||
textureIndex = (short) (textureIndex + numPrimitiveTextureCoords);
|
||||
}
|
||||
|
||||
// Combine indices and look up in pair map.
|
||||
ShortPair indexPair = new ShortPair(Short.valueOf(vertexIndex), Short.valueOf(textureIndex));
|
||||
Short combinedIndex = vertexTexCoordMap.get(indexPair);
|
||||
if (combinedIndex == null) {
|
||||
short numIndexPairs = (short) vertexTexCoordMap.size();
|
||||
vertexTexCoordMap.put(indexPair, numIndexPairs);
|
||||
return numIndexPairs;
|
||||
} else {
|
||||
return combinedIndex.shortValue();
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
// Failure to parse coordinates as shorts
|
||||
return INVALID_INDEX;
|
||||
}
|
||||
}
|
||||
|
||||
// Note: it is assumed that face list occurs AFTER vertex and texture coordinate lists finish in
|
||||
// the obj file format.
|
||||
private void parseFace(String[] linePieces) {
|
||||
if (linePieces.length < 4) {
|
||||
System.out.println("Malformed face index list: there must be at least 3 indices per face");
|
||||
return;
|
||||
}
|
||||
|
||||
short[] faceIndices = new short[linePieces.length - 1];
|
||||
boolean success = true;
|
||||
for (int i = 1; i < linePieces.length; i++) {
|
||||
short faceIndex = parseAndProcessCombinedVertexCoord(linePieces[i]);
|
||||
|
||||
if (faceIndex < 0) {
|
||||
System.out.println(faceIndex);
|
||||
System.out.println("Malformed face index: " + linePieces[i]);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
faceIndices[i - 1] = faceIndex;
|
||||
}
|
||||
|
||||
if (success) {
|
||||
numPrimitiveFaces++;
|
||||
// Manually triangulate the face under the assumption that the points are coplanar, the poly
|
||||
// is convex, and the points are listed in either clockwise or anti-clockwise orientation.
|
||||
for (int i = 1; i < faceIndices.length - 1; i++) {
|
||||
// We use a triangle fan here, so first point is part of all triangles
|
||||
finalizedTriangles.add(faceIndices[0]);
|
||||
finalizedTriangles.add(faceIndices[i]);
|
||||
finalizedTriangles.add(faceIndices[i + 1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Iterate over map and reconstruct proper vertex/texture coordinate pairings.
|
||||
private boolean constructFinalCoordinatesFromMap() {
|
||||
final int numIndexPairs = vertexTexCoordMap.size();
|
||||
// XYZ vertices and UV texture coordinates
|
||||
finalizedVertices = new float[POSITIONS_COORDS_PER_VERTEX * numIndexPairs];
|
||||
finalizedTextureCoords = new float[TEXTURE_COORDS_PER_VERTEX * numIndexPairs];
|
||||
try {
|
||||
for (Map.Entry<ShortPair, Short> entry : vertexTexCoordMap.entrySet()) {
|
||||
ShortPair indexPair = entry.getKey();
|
||||
short rawVertexIndex = indexPair.getFirst().shortValue();
|
||||
short rawTexCoordIndex = indexPair.getSecond().shortValue();
|
||||
short finalIndex = entry.getValue().shortValue();
|
||||
for (int i = 0; i < POSITIONS_COORDS_PER_VERTEX; i++) {
|
||||
finalizedVertices[POSITIONS_COORDS_PER_VERTEX * finalIndex + i]
|
||||
= vertices.get(rawVertexIndex * POSITIONS_COORDS_PER_VERTEX + i);
|
||||
}
|
||||
for (int i = 0; i < TEXTURE_COORDS_PER_VERTEX; i++) {
|
||||
finalizedTextureCoords[TEXTURE_COORDS_PER_VERTEX * finalIndex + i]
|
||||
= textureCoords.get(rawTexCoordIndex * TEXTURE_COORDS_PER_VERTEX + i);
|
||||
}
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
System.out.println("Malformed index in vertex/texture coordinate mapping.");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the vertex position coordinate list (x1, y1, z1, x2, y2, z2, ...) after a successful
|
||||
* call to parse().
|
||||
*/
|
||||
public float[] getVertices() {
|
||||
return finalizedVertices;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the vertex texture coordinate list (u1, v1, u2, v2, ...) after a successful call to
|
||||
* parse().
|
||||
*/
|
||||
public float[] getTextureCoords() {
|
||||
return finalizedTextureCoords;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the list of indices (a1, b1, c1, a2, b2, c2, ...) after a successful call to parse().
|
||||
* Each (a, b, c) triplet specifies a triangle to be rendered, with a, b, and c Short objects used
|
||||
* to index into the coordinates returned by getVertices() and getTextureCoords().<p></p>
|
||||
* For example, a Short index representing 5 should be used to index into vertices[15],
|
||||
* vertices[16], and vertices[17], as well as textureCoords[10] and textureCoords[11].
|
||||
*/
|
||||
public ArrayList<Short> getTriangles() {
|
||||
return finalizedTriangles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to locate and read the specified .obj file, and parse it accordingly. None of the
|
||||
* getter functions in this class will return valid results until a value of true is returned
|
||||
* from this function.
|
||||
* @return true on success.
|
||||
*/
|
||||
public boolean parse() {
|
||||
boolean success = true;
|
||||
BufferedReader reader = null;
|
||||
try {
|
||||
reader = Files.newBufferedReader(Paths.get(fileName), UTF_8);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
// Skip over lines with no characters
|
||||
if (line.length() < 1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Ignore comment lines entirely
|
||||
if (line.charAt(0) == '#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Split into pieces based on whitespace, and process according to first command piece
|
||||
String[] linePieces = line.split(" +");
|
||||
switch (linePieces[0]) {
|
||||
case "v":
|
||||
// Add vertex
|
||||
if (startedProcessingFaces) {
|
||||
throw new IOException("Vertices must all be declared before faces in obj files.");
|
||||
}
|
||||
parseVertex(linePieces);
|
||||
break;
|
||||
case "vt":
|
||||
// Add texture coordinate
|
||||
if (startedProcessingFaces) {
|
||||
throw new IOException(
|
||||
"Texture coordinates must all be declared before faces in obj files.");
|
||||
}
|
||||
parseTextureCoordinate(linePieces);
|
||||
break;
|
||||
case "f":
|
||||
// Vertex and texture coordinate lists should be locked into place by now
|
||||
if (!startedProcessingFaces) {
|
||||
startedProcessingFaces = true;
|
||||
numPrimitiveVertices = vertices.size() / POSITIONS_COORDS_PER_VERTEX;
|
||||
numPrimitiveTextureCoords = textureCoords.size() / TEXTURE_COORDS_PER_VERTEX;
|
||||
}
|
||||
// Add face
|
||||
parseFace(linePieces);
|
||||
break;
|
||||
default:
|
||||
// Unknown or unused directive: ignoring
|
||||
// Note: We do not yet process vertex normals or curves, so we ignore {vp, vn, s}
|
||||
// Note: We assume only a single object, so we ignore {g, o}
|
||||
// Note: We also assume a single texture, which we process independently, so we ignore
|
||||
// {mtllib, usemtl}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If we made it all the way through, then we have a vertex-to-tex-coord pair mapping, so
|
||||
// construct our final vertex and texture coordinate lists now.
|
||||
success = constructFinalCoordinatesFromMap();
|
||||
|
||||
} catch (IOException e) {
|
||||
success = false;
|
||||
System.out.println("Failure to parse obj file: " + e.toString());
|
||||
} finally {
|
||||
try {
|
||||
if (reader != null) {
|
||||
reader.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
System.out.println("Couldn't close reader");
|
||||
}
|
||||
}
|
||||
if (success) {
|
||||
debugLogString("Successfully parsed " + numPrimitiveVertices + " vertices and "
|
||||
+ numPrimitiveTextureCoords + " texture coordinates into " + vertexTexCoordMap.size()
|
||||
+ " combined vertices and " + numPrimitiveFaces + " faces, represented as a mesh of "
|
||||
+ finalizedTriangles.size() / 3 + " triangles.");
|
||||
}
|
||||
return success;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2021 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# The SimpleObjParser expects the obj commands to follow v/vt/f order. This
|
||||
# little script will read all the obj files in a directory and sort the
|
||||
# existing obj commands inside them to also follow this order (so all v lines
|
||||
# will appear before all vt lines, which will appear before all f lines).
|
||||
|
||||
# Usage: ./obj_cleanup.sh input_folder output_folder
|
||||
# input_folder and output_folder paths can be absolute or relative.
|
||||
|
||||
input_folder=$1
|
||||
output_folder=$2
|
||||
if [[ "${input_folder}" == "" ]]; then
|
||||
echo "input_folder must be defined. Usage: ./obj_cleanup.sh input_folder output_folder"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${output_folder}" == "" ]]; then
|
||||
echo "output_folder must be defined. Usage: ./obj_cleanup.sh input_folder output_folder"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Find all the obj files and remove the directory name
|
||||
# Interestingly, piping | sed 's!.obj!! also removed the extension obj too.
|
||||
find "${input_folder}" -name "*.obj" | sed 's!.*/!!' | sort |
|
||||
while IFS= read -r filename; do
|
||||
echo "Clean up ${filename}"
|
||||
cat "${input_folder}/${filename}" | grep 'v ' > "${output_folder}/${filename}"
|
||||
cat "${input_folder}/${filename}" | grep 'vt ' >> "${output_folder}/${filename}"
|
||||
cat "${input_folder}/${filename}" | grep 'f ' >> "${output_folder}/${filename}"
|
||||
done
|
||||
@@ -23,22 +23,11 @@ node {
|
||||
output_stream: "VIDEO_PRESTREAM:input_video_header"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "LocalFileContentsCalculator"
|
||||
input_side_packet: "FILE_PATH:0:box_landmark_model_path"
|
||||
output_side_packet: "CONTENTS:0:box_landmark_model_blob"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "TfLiteModelCalculator"
|
||||
input_side_packet: "MODEL_BLOB:box_landmark_model_blob"
|
||||
output_side_packet: "MODEL:box_landmark_model"
|
||||
}
|
||||
|
||||
# Run Objectron subgraph.
|
||||
node {
|
||||
calculator: "ObjectronCpuSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_side_packet: "MODEL:box_landmark_model"
|
||||
input_side_packet: "MODEL_PATH:box_landmark_model_path"
|
||||
input_side_packet: "LABELS_CSV:allowed_labels"
|
||||
input_side_packet: "MAX_NUM_OBJECTS:max_num_objects"
|
||||
output_stream: "MULTI_LANDMARKS:box_landmarks"
|
||||
|
||||
Reference in New Issue
Block a user