Project import generated by Copybara.
GitOrigin-RevId: 612e50bb8db2ec3dc1c30049372d87a80c3848db
This commit is contained in:
MediaPipe Team
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
cc_library(
|
||||
name = "face_landmarks_to_render_data_calculator",
|
||||
srcs = ["face_landmarks_to_render_data_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_options_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:location_data_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util:color_cc_proto",
|
||||
"//mediapipe/util:render_data_cc_proto",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "absl/strings/str_join.h"
|
||||
#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.h"
|
||||
#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_options.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/location_data.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/util/color.pb.h"
|
||||
#include "mediapipe/util/render_data.pb.h"
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int kNumFaceLandmarkConnections = 124;
|
||||
// Pairs of landmark indices to be rendered with connections.
|
||||
constexpr int kFaceLandmarkConnections[] = {
|
||||
// Lips.
|
||||
61, 146, 146, 91, 91, 181, 181, 84, 84, 17, 17, 314, 314, 405, 405, 321,
|
||||
321, 375, 375, 291, 61, 185, 185, 40, 40, 39, 39, 37, 37, 0, 0, 267, 267,
|
||||
269, 269, 270, 270, 409, 409, 291, 78, 95, 95, 88, 88, 178, 178, 87, 87, 14,
|
||||
14, 317, 317, 402, 402, 318, 318, 324, 324, 308, 78, 191, 191, 80, 80, 81,
|
||||
81, 82, 82, 13, 13, 312, 312, 311, 311, 310, 310, 415, 415, 308,
|
||||
// Left eye.
|
||||
33, 7, 7, 163, 163, 144, 144, 145, 145, 153, 153, 154, 154, 155, 155, 133,
|
||||
33, 246, 246, 161, 161, 160, 160, 159, 159, 158, 158, 157, 157, 173, 173,
|
||||
133,
|
||||
// Left eyebrow.
|
||||
46, 53, 53, 52, 52, 65, 65, 55, 70, 63, 63, 105, 105, 66, 66, 107,
|
||||
// Right eye.
|
||||
263, 249, 249, 390, 390, 373, 373, 374, 374, 380, 380, 381, 381, 382, 382,
|
||||
362, 263, 466, 466, 388, 388, 387, 387, 386, 386, 385, 385, 384, 384, 398,
|
||||
398, 362,
|
||||
// Right eyebrow.
|
||||
276, 283, 283, 282, 282, 295, 295, 285, 300, 293, 293, 334, 334, 296, 296,
|
||||
336,
|
||||
// Face oval.
|
||||
10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356,
|
||||
454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378,
|
||||
378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150,
|
||||
136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162,
|
||||
21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10};
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator that converts face landmarks to RenderData proto for
|
||||
// visualization. Ignores landmark_connections specified in
|
||||
// LandmarksToRenderDataCalculatorOptions, if any, and always uses a fixed set
|
||||
// of landmark connections specific to face landmark (defined in
|
||||
// kFaceLandmarkConnections[] above).
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "FaceLandmarksToRenderDataCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:landmarks"
|
||||
// output_stream: "RENDER_DATA:render_data"
|
||||
// options {
|
||||
// [LandmarksToRenderDataCalculatorOptions.ext] {
|
||||
// landmark_color { r: 0 g: 255 b: 0 }
|
||||
// connection_color { r: 0 g: 255 b: 0 }
|
||||
// thickness: 4.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class FaceLandmarksToRenderDataCalculator
|
||||
: public LandmarksToRenderDataCalculator {
|
||||
public:
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(FaceLandmarksToRenderDataCalculator);
|
||||
|
||||
::mediapipe::Status FaceLandmarksToRenderDataCalculator::Open(
|
||||
CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
options_ = cc->Options<mediapipe::LandmarksToRenderDataCalculatorOptions>();
|
||||
|
||||
for (int i = 0; i < kNumFaceLandmarkConnections; ++i) {
|
||||
landmark_connections_.push_back(kFaceLandmarkConnections[i * 2]);
|
||||
landmark_connections_.push_back(kFaceLandmarkConnections[i * 2 + 1]);
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
@@ -29,6 +29,7 @@ cc_library(
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:rect_to_render_data_calculator",
|
||||
"//mediapipe/graphs/face_mesh/calculators:face_landmarks_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -48,268 +48,14 @@ node {
|
||||
|
||||
# Converts landmarks to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LandmarksToRenderDataCalculator"
|
||||
calculator: "FaceLandmarksToRenderDataCalculator"
|
||||
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||
output_stream: "RENDER_DATA:landmark_render_data"
|
||||
output_stream: "RENDER_DATA:landmarks_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||
# Lips.
|
||||
landmark_connections: 61
|
||||
landmark_connections: 146
|
||||
landmark_connections: 146
|
||||
landmark_connections: 91
|
||||
landmark_connections: 91
|
||||
landmark_connections: 181
|
||||
landmark_connections: 181
|
||||
landmark_connections: 84
|
||||
landmark_connections: 84
|
||||
landmark_connections: 17
|
||||
landmark_connections: 17
|
||||
landmark_connections: 314
|
||||
landmark_connections: 314
|
||||
landmark_connections: 405
|
||||
landmark_connections: 405
|
||||
landmark_connections: 321
|
||||
landmark_connections: 321
|
||||
landmark_connections: 375
|
||||
landmark_connections: 375
|
||||
landmark_connections: 291
|
||||
landmark_connections: 61
|
||||
landmark_connections: 185
|
||||
landmark_connections: 185
|
||||
landmark_connections: 40
|
||||
landmark_connections: 40
|
||||
landmark_connections: 39
|
||||
landmark_connections: 39
|
||||
landmark_connections: 37
|
||||
landmark_connections: 37
|
||||
landmark_connections: 0
|
||||
landmark_connections: 0
|
||||
landmark_connections: 267
|
||||
landmark_connections: 267
|
||||
landmark_connections: 269
|
||||
landmark_connections: 269
|
||||
landmark_connections: 270
|
||||
landmark_connections: 270
|
||||
landmark_connections: 409
|
||||
landmark_connections: 409
|
||||
landmark_connections: 291
|
||||
landmark_connections: 78
|
||||
landmark_connections: 95
|
||||
landmark_connections: 95
|
||||
landmark_connections: 88
|
||||
landmark_connections: 88
|
||||
landmark_connections: 178
|
||||
landmark_connections: 178
|
||||
landmark_connections: 87
|
||||
landmark_connections: 87
|
||||
landmark_connections: 14
|
||||
landmark_connections: 14
|
||||
landmark_connections: 317
|
||||
landmark_connections: 317
|
||||
landmark_connections: 402
|
||||
landmark_connections: 402
|
||||
landmark_connections: 318
|
||||
landmark_connections: 318
|
||||
landmark_connections: 324
|
||||
landmark_connections: 324
|
||||
landmark_connections: 308
|
||||
landmark_connections: 78
|
||||
landmark_connections: 191
|
||||
landmark_connections: 191
|
||||
landmark_connections: 80
|
||||
landmark_connections: 80
|
||||
landmark_connections: 81
|
||||
landmark_connections: 81
|
||||
landmark_connections: 82
|
||||
landmark_connections: 82
|
||||
landmark_connections: 13
|
||||
landmark_connections: 13
|
||||
landmark_connections: 312
|
||||
landmark_connections: 312
|
||||
landmark_connections: 311
|
||||
landmark_connections: 311
|
||||
landmark_connections: 310
|
||||
landmark_connections: 310
|
||||
landmark_connections: 415
|
||||
landmark_connections: 415
|
||||
landmark_connections: 308
|
||||
# Left eye.
|
||||
landmark_connections: 33
|
||||
landmark_connections: 7
|
||||
landmark_connections: 7
|
||||
landmark_connections: 163
|
||||
landmark_connections: 163
|
||||
landmark_connections: 144
|
||||
landmark_connections: 144
|
||||
landmark_connections: 145
|
||||
landmark_connections: 145
|
||||
landmark_connections: 153
|
||||
landmark_connections: 153
|
||||
landmark_connections: 154
|
||||
landmark_connections: 154
|
||||
landmark_connections: 155
|
||||
landmark_connections: 155
|
||||
landmark_connections: 133
|
||||
landmark_connections: 33
|
||||
landmark_connections: 246
|
||||
landmark_connections: 246
|
||||
landmark_connections: 161
|
||||
landmark_connections: 161
|
||||
landmark_connections: 160
|
||||
landmark_connections: 160
|
||||
landmark_connections: 159
|
||||
landmark_connections: 159
|
||||
landmark_connections: 158
|
||||
landmark_connections: 158
|
||||
landmark_connections: 157
|
||||
landmark_connections: 157
|
||||
landmark_connections: 173
|
||||
landmark_connections: 173
|
||||
landmark_connections: 133
|
||||
# Left eyebrow.
|
||||
landmark_connections: 46
|
||||
landmark_connections: 53
|
||||
landmark_connections: 53
|
||||
landmark_connections: 52
|
||||
landmark_connections: 52
|
||||
landmark_connections: 65
|
||||
landmark_connections: 65
|
||||
landmark_connections: 55
|
||||
landmark_connections: 70
|
||||
landmark_connections: 63
|
||||
landmark_connections: 63
|
||||
landmark_connections: 105
|
||||
landmark_connections: 105
|
||||
landmark_connections: 66
|
||||
landmark_connections: 66
|
||||
landmark_connections: 107
|
||||
# Right eye.
|
||||
landmark_connections: 263
|
||||
landmark_connections: 249
|
||||
landmark_connections: 249
|
||||
landmark_connections: 390
|
||||
landmark_connections: 390
|
||||
landmark_connections: 373
|
||||
landmark_connections: 373
|
||||
landmark_connections: 374
|
||||
landmark_connections: 374
|
||||
landmark_connections: 380
|
||||
landmark_connections: 380
|
||||
landmark_connections: 381
|
||||
landmark_connections: 381
|
||||
landmark_connections: 382
|
||||
landmark_connections: 382
|
||||
landmark_connections: 362
|
||||
landmark_connections: 263
|
||||
landmark_connections: 466
|
||||
landmark_connections: 466
|
||||
landmark_connections: 388
|
||||
landmark_connections: 388
|
||||
landmark_connections: 387
|
||||
landmark_connections: 387
|
||||
landmark_connections: 386
|
||||
landmark_connections: 386
|
||||
landmark_connections: 385
|
||||
landmark_connections: 385
|
||||
landmark_connections: 384
|
||||
landmark_connections: 384
|
||||
landmark_connections: 398
|
||||
landmark_connections: 398
|
||||
landmark_connections: 362
|
||||
# Right eyebrow.
|
||||
landmark_connections: 276
|
||||
landmark_connections: 283
|
||||
landmark_connections: 283
|
||||
landmark_connections: 282
|
||||
landmark_connections: 282
|
||||
landmark_connections: 295
|
||||
landmark_connections: 295
|
||||
landmark_connections: 285
|
||||
landmark_connections: 300
|
||||
landmark_connections: 293
|
||||
landmark_connections: 293
|
||||
landmark_connections: 334
|
||||
landmark_connections: 334
|
||||
landmark_connections: 296
|
||||
landmark_connections: 296
|
||||
landmark_connections: 336
|
||||
# Face oval.
|
||||
landmark_connections: 10
|
||||
landmark_connections: 338
|
||||
landmark_connections: 338
|
||||
landmark_connections: 297
|
||||
landmark_connections: 297
|
||||
landmark_connections: 332
|
||||
landmark_connections: 332
|
||||
landmark_connections: 284
|
||||
landmark_connections: 284
|
||||
landmark_connections: 251
|
||||
landmark_connections: 251
|
||||
landmark_connections: 389
|
||||
landmark_connections: 389
|
||||
landmark_connections: 356
|
||||
landmark_connections: 356
|
||||
landmark_connections: 454
|
||||
landmark_connections: 454
|
||||
landmark_connections: 323
|
||||
landmark_connections: 323
|
||||
landmark_connections: 361
|
||||
landmark_connections: 361
|
||||
landmark_connections: 288
|
||||
landmark_connections: 288
|
||||
landmark_connections: 397
|
||||
landmark_connections: 397
|
||||
landmark_connections: 365
|
||||
landmark_connections: 365
|
||||
landmark_connections: 379
|
||||
landmark_connections: 379
|
||||
landmark_connections: 378
|
||||
landmark_connections: 378
|
||||
landmark_connections: 400
|
||||
landmark_connections: 400
|
||||
landmark_connections: 377
|
||||
landmark_connections: 377
|
||||
landmark_connections: 152
|
||||
landmark_connections: 152
|
||||
landmark_connections: 148
|
||||
landmark_connections: 148
|
||||
landmark_connections: 176
|
||||
landmark_connections: 176
|
||||
landmark_connections: 149
|
||||
landmark_connections: 149
|
||||
landmark_connections: 150
|
||||
landmark_connections: 150
|
||||
landmark_connections: 136
|
||||
landmark_connections: 136
|
||||
landmark_connections: 172
|
||||
landmark_connections: 172
|
||||
landmark_connections: 58
|
||||
landmark_connections: 58
|
||||
landmark_connections: 132
|
||||
landmark_connections: 132
|
||||
landmark_connections: 93
|
||||
landmark_connections: 93
|
||||
landmark_connections: 234
|
||||
landmark_connections: 234
|
||||
landmark_connections: 127
|
||||
landmark_connections: 127
|
||||
landmark_connections: 162
|
||||
landmark_connections: 162
|
||||
landmark_connections: 21
|
||||
landmark_connections: 21
|
||||
landmark_connections: 54
|
||||
landmark_connections: 54
|
||||
landmark_connections: 103
|
||||
landmark_connections: 103
|
||||
landmark_connections: 67
|
||||
landmark_connections: 67
|
||||
landmark_connections: 109
|
||||
landmark_connections: 109
|
||||
landmark_connections: 10
|
||||
landmark_color { r: 255 g: 0 b: 0 }
|
||||
connection_color { r: 0 g: 255 b: 0 }
|
||||
thickness: 1.5
|
||||
thickness: 2
|
||||
visualize_landmark_depth: false
|
||||
}
|
||||
}
|
||||
@@ -320,7 +66,7 @@ node {
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopRenderDataCalculator"
|
||||
input_stream: "ITEM:landmark_render_data"
|
||||
input_stream: "ITEM:landmarks_render_data"
|
||||
input_stream: "BATCH_END:landmark_timestamp"
|
||||
output_stream: "ITERABLE:multi_face_landmarks_render_data"
|
||||
}
|
||||
|
||||
@@ -48,265 +48,11 @@ node {
|
||||
|
||||
# Converts landmarks to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LandmarksToRenderDataCalculator"
|
||||
calculator: "FaceLandmarksToRenderDataCalculator"
|
||||
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||
output_stream: "RENDER_DATA:landmarks_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||
# Lips.
|
||||
landmark_connections: 61
|
||||
landmark_connections: 146
|
||||
landmark_connections: 146
|
||||
landmark_connections: 91
|
||||
landmark_connections: 91
|
||||
landmark_connections: 181
|
||||
landmark_connections: 181
|
||||
landmark_connections: 84
|
||||
landmark_connections: 84
|
||||
landmark_connections: 17
|
||||
landmark_connections: 17
|
||||
landmark_connections: 314
|
||||
landmark_connections: 314
|
||||
landmark_connections: 405
|
||||
landmark_connections: 405
|
||||
landmark_connections: 321
|
||||
landmark_connections: 321
|
||||
landmark_connections: 375
|
||||
landmark_connections: 375
|
||||
landmark_connections: 291
|
||||
landmark_connections: 61
|
||||
landmark_connections: 185
|
||||
landmark_connections: 185
|
||||
landmark_connections: 40
|
||||
landmark_connections: 40
|
||||
landmark_connections: 39
|
||||
landmark_connections: 39
|
||||
landmark_connections: 37
|
||||
landmark_connections: 37
|
||||
landmark_connections: 0
|
||||
landmark_connections: 0
|
||||
landmark_connections: 267
|
||||
landmark_connections: 267
|
||||
landmark_connections: 269
|
||||
landmark_connections: 269
|
||||
landmark_connections: 270
|
||||
landmark_connections: 270
|
||||
landmark_connections: 409
|
||||
landmark_connections: 409
|
||||
landmark_connections: 291
|
||||
landmark_connections: 78
|
||||
landmark_connections: 95
|
||||
landmark_connections: 95
|
||||
landmark_connections: 88
|
||||
landmark_connections: 88
|
||||
landmark_connections: 178
|
||||
landmark_connections: 178
|
||||
landmark_connections: 87
|
||||
landmark_connections: 87
|
||||
landmark_connections: 14
|
||||
landmark_connections: 14
|
||||
landmark_connections: 317
|
||||
landmark_connections: 317
|
||||
landmark_connections: 402
|
||||
landmark_connections: 402
|
||||
landmark_connections: 318
|
||||
landmark_connections: 318
|
||||
landmark_connections: 324
|
||||
landmark_connections: 324
|
||||
landmark_connections: 308
|
||||
landmark_connections: 78
|
||||
landmark_connections: 191
|
||||
landmark_connections: 191
|
||||
landmark_connections: 80
|
||||
landmark_connections: 80
|
||||
landmark_connections: 81
|
||||
landmark_connections: 81
|
||||
landmark_connections: 82
|
||||
landmark_connections: 82
|
||||
landmark_connections: 13
|
||||
landmark_connections: 13
|
||||
landmark_connections: 312
|
||||
landmark_connections: 312
|
||||
landmark_connections: 311
|
||||
landmark_connections: 311
|
||||
landmark_connections: 310
|
||||
landmark_connections: 310
|
||||
landmark_connections: 415
|
||||
landmark_connections: 415
|
||||
landmark_connections: 308
|
||||
# Left eye.
|
||||
landmark_connections: 33
|
||||
landmark_connections: 7
|
||||
landmark_connections: 7
|
||||
landmark_connections: 163
|
||||
landmark_connections: 163
|
||||
landmark_connections: 144
|
||||
landmark_connections: 144
|
||||
landmark_connections: 145
|
||||
landmark_connections: 145
|
||||
landmark_connections: 153
|
||||
landmark_connections: 153
|
||||
landmark_connections: 154
|
||||
landmark_connections: 154
|
||||
landmark_connections: 155
|
||||
landmark_connections: 155
|
||||
landmark_connections: 133
|
||||
landmark_connections: 33
|
||||
landmark_connections: 246
|
||||
landmark_connections: 246
|
||||
landmark_connections: 161
|
||||
landmark_connections: 161
|
||||
landmark_connections: 160
|
||||
landmark_connections: 160
|
||||
landmark_connections: 159
|
||||
landmark_connections: 159
|
||||
landmark_connections: 158
|
||||
landmark_connections: 158
|
||||
landmark_connections: 157
|
||||
landmark_connections: 157
|
||||
landmark_connections: 173
|
||||
landmark_connections: 173
|
||||
landmark_connections: 133
|
||||
# Left eyebrow.
|
||||
landmark_connections: 46
|
||||
landmark_connections: 53
|
||||
landmark_connections: 53
|
||||
landmark_connections: 52
|
||||
landmark_connections: 52
|
||||
landmark_connections: 65
|
||||
landmark_connections: 65
|
||||
landmark_connections: 55
|
||||
landmark_connections: 70
|
||||
landmark_connections: 63
|
||||
landmark_connections: 63
|
||||
landmark_connections: 105
|
||||
landmark_connections: 105
|
||||
landmark_connections: 66
|
||||
landmark_connections: 66
|
||||
landmark_connections: 107
|
||||
# Right eye.
|
||||
landmark_connections: 263
|
||||
landmark_connections: 249
|
||||
landmark_connections: 249
|
||||
landmark_connections: 390
|
||||
landmark_connections: 390
|
||||
landmark_connections: 373
|
||||
landmark_connections: 373
|
||||
landmark_connections: 374
|
||||
landmark_connections: 374
|
||||
landmark_connections: 380
|
||||
landmark_connections: 380
|
||||
landmark_connections: 381
|
||||
landmark_connections: 381
|
||||
landmark_connections: 382
|
||||
landmark_connections: 382
|
||||
landmark_connections: 362
|
||||
landmark_connections: 263
|
||||
landmark_connections: 466
|
||||
landmark_connections: 466
|
||||
landmark_connections: 388
|
||||
landmark_connections: 388
|
||||
landmark_connections: 387
|
||||
landmark_connections: 387
|
||||
landmark_connections: 386
|
||||
landmark_connections: 386
|
||||
landmark_connections: 385
|
||||
landmark_connections: 385
|
||||
landmark_connections: 384
|
||||
landmark_connections: 384
|
||||
landmark_connections: 398
|
||||
landmark_connections: 398
|
||||
landmark_connections: 362
|
||||
# Right eyebrow.
|
||||
landmark_connections: 276
|
||||
landmark_connections: 283
|
||||
landmark_connections: 283
|
||||
landmark_connections: 282
|
||||
landmark_connections: 282
|
||||
landmark_connections: 295
|
||||
landmark_connections: 295
|
||||
landmark_connections: 285
|
||||
landmark_connections: 300
|
||||
landmark_connections: 293
|
||||
landmark_connections: 293
|
||||
landmark_connections: 334
|
||||
landmark_connections: 334
|
||||
landmark_connections: 296
|
||||
landmark_connections: 296
|
||||
landmark_connections: 336
|
||||
# Face oval.
|
||||
landmark_connections: 10
|
||||
landmark_connections: 338
|
||||
landmark_connections: 338
|
||||
landmark_connections: 297
|
||||
landmark_connections: 297
|
||||
landmark_connections: 332
|
||||
landmark_connections: 332
|
||||
landmark_connections: 284
|
||||
landmark_connections: 284
|
||||
landmark_connections: 251
|
||||
landmark_connections: 251
|
||||
landmark_connections: 389
|
||||
landmark_connections: 389
|
||||
landmark_connections: 356
|
||||
landmark_connections: 356
|
||||
landmark_connections: 454
|
||||
landmark_connections: 454
|
||||
landmark_connections: 323
|
||||
landmark_connections: 323
|
||||
landmark_connections: 361
|
||||
landmark_connections: 361
|
||||
landmark_connections: 288
|
||||
landmark_connections: 288
|
||||
landmark_connections: 397
|
||||
landmark_connections: 397
|
||||
landmark_connections: 365
|
||||
landmark_connections: 365
|
||||
landmark_connections: 379
|
||||
landmark_connections: 379
|
||||
landmark_connections: 378
|
||||
landmark_connections: 378
|
||||
landmark_connections: 400
|
||||
landmark_connections: 400
|
||||
landmark_connections: 377
|
||||
landmark_connections: 377
|
||||
landmark_connections: 152
|
||||
landmark_connections: 152
|
||||
landmark_connections: 148
|
||||
landmark_connections: 148
|
||||
landmark_connections: 176
|
||||
landmark_connections: 176
|
||||
landmark_connections: 149
|
||||
landmark_connections: 149
|
||||
landmark_connections: 150
|
||||
landmark_connections: 150
|
||||
landmark_connections: 136
|
||||
landmark_connections: 136
|
||||
landmark_connections: 172
|
||||
landmark_connections: 172
|
||||
landmark_connections: 58
|
||||
landmark_connections: 58
|
||||
landmark_connections: 132
|
||||
landmark_connections: 132
|
||||
landmark_connections: 93
|
||||
landmark_connections: 93
|
||||
landmark_connections: 234
|
||||
landmark_connections: 234
|
||||
landmark_connections: 127
|
||||
landmark_connections: 127
|
||||
landmark_connections: 162
|
||||
landmark_connections: 162
|
||||
landmark_connections: 21
|
||||
landmark_connections: 21
|
||||
landmark_connections: 54
|
||||
landmark_connections: 54
|
||||
landmark_connections: 103
|
||||
landmark_connections: 103
|
||||
landmark_connections: 67
|
||||
landmark_connections: 67
|
||||
landmark_connections: 109
|
||||
landmark_connections: 109
|
||||
landmark_connections: 10
|
||||
landmark_color { r: 255 g: 0 b: 0 }
|
||||
connection_color { r: 0 g: 255 b: 0 }
|
||||
thickness: 2
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -12,20 +12,18 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
cc_library(
|
||||
name = "hand_detections_to_rects_calculator",
|
||||
srcs = ["hand_detections_to_rects_calculator.cc"],
|
||||
name = "hand_landmarks_to_rect_calculator",
|
||||
srcs = ["hand_landmarks_to_rect_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_options_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:location_data_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cmath>
|
||||
|
||||
#include "mediapipe/calculators/util/detections_to_rects_calculator.h"
|
||||
#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_options.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/location_data.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
// Indices of joints used for computing the rotation of the output rectangle
|
||||
// from detection box with keypoints.
|
||||
constexpr int kWristJoint = 0;
|
||||
constexpr int kMiddleFingerPIPJoint = 6;
|
||||
constexpr int kIndexFingerPIPJoint = 4;
|
||||
constexpr int kRingFingerPIPJoint = 8;
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator that converts Hand detection to a bounding box NormalizedRect.
|
||||
// The calculator overwrites the default logic of DetectionsToRectsCalculator
|
||||
// for rotating the detection bounding to a rectangle. The rotation angle is
|
||||
// computed based on 1) the wrist joint and 2) the average of PIP joints of
|
||||
// index finger, middle finger and ring finger. After rotation, the vector from
|
||||
// the wrist to the mean of PIP joints is expected to be vertical with wrist at
|
||||
// the bottom and the mean of PIP joints at the top.
|
||||
class HandDetectionsToRectsCalculator : public DetectionsToRectsCalculator {
|
||||
public:
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
RET_CHECK(cc->Inputs().HasTag(kImageSizeTag))
|
||||
<< "Image size is required to calculate rotated rect";
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
target_angle_ = M_PI * 0.5f;
|
||||
rotate_ = true;
|
||||
options_ = cc->Options<DetectionsToRectsCalculatorOptions>();
|
||||
output_zero_rect_for_empty_detections_ =
|
||||
options_.output_zero_rect_for_empty_detections();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
::mediapipe::Status ComputeRotation(const ::mediapipe::Detection& detection,
|
||||
const DetectionSpec& detection_spec,
|
||||
float* rotation) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(HandDetectionsToRectsCalculator);
|
||||
|
||||
::mediapipe::Status HandDetectionsToRectsCalculator::ComputeRotation(
|
||||
const Detection& detection, const DetectionSpec& detection_spec,
|
||||
float* rotation) {
|
||||
const auto& location_data = detection.location_data();
|
||||
const auto& image_size = detection_spec.image_size;
|
||||
RET_CHECK(image_size) << "Image size is required to calculate rotation";
|
||||
|
||||
const float x0 =
|
||||
location_data.relative_keypoints(kWristJoint).x() * image_size->first;
|
||||
const float y0 =
|
||||
location_data.relative_keypoints(kWristJoint).y() * image_size->second;
|
||||
|
||||
float x1 = (location_data.relative_keypoints(kIndexFingerPIPJoint).x() +
|
||||
location_data.relative_keypoints(kRingFingerPIPJoint).x()) /
|
||||
2.f;
|
||||
float y1 = (location_data.relative_keypoints(kIndexFingerPIPJoint).y() +
|
||||
location_data.relative_keypoints(kRingFingerPIPJoint).y()) /
|
||||
2.f;
|
||||
x1 = (x1 + location_data.relative_keypoints(kMiddleFingerPIPJoint).x()) /
|
||||
2.f * image_size->first;
|
||||
y1 = (y1 + location_data.relative_keypoints(kMiddleFingerPIPJoint).y()) /
|
||||
2.f * image_size->second;
|
||||
|
||||
*rotation = NormalizeRadians(target_angle_ - std::atan2(-(y1 - y0), x1 - x0));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include <cmath>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_options.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
|
||||
constexpr char kNormRectTag[] = "NORM_RECT";
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
constexpr int kWristJoint = 0;
|
||||
constexpr int kMiddleFingerPIPJoint = 6;
|
||||
constexpr int kIndexFingerPIPJoint = 4;
|
||||
constexpr int kRingFingerPIPJoint = 8;
|
||||
constexpr float kTargetAngle = M_PI * 0.5f;
|
||||
|
||||
inline float NormalizeRadians(float angle) {
|
||||
return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI));
|
||||
}
|
||||
|
||||
float ComputeRotation(const NormalizedLandmarkList& landmarks,
|
||||
const std::pair<int, int>& image_size) {
|
||||
const float x0 = landmarks.landmark(kWristJoint).x() * image_size.first;
|
||||
const float y0 = landmarks.landmark(kWristJoint).y() * image_size.second;
|
||||
|
||||
float x1 = (landmarks.landmark(kIndexFingerPIPJoint).x() +
|
||||
landmarks.landmark(kRingFingerPIPJoint).x()) /
|
||||
2.f;
|
||||
float y1 = (landmarks.landmark(kIndexFingerPIPJoint).y() +
|
||||
landmarks.landmark(kRingFingerPIPJoint).y()) /
|
||||
2.f;
|
||||
x1 = (x1 + landmarks.landmark(kMiddleFingerPIPJoint).x()) / 2.f *
|
||||
image_size.first;
|
||||
y1 = (y1 + landmarks.landmark(kMiddleFingerPIPJoint).y()) / 2.f *
|
||||
image_size.second;
|
||||
|
||||
const float rotation =
|
||||
NormalizeRadians(kTargetAngle - std::atan2(-(y1 - y0), x1 - x0));
|
||||
return rotation;
|
||||
}
|
||||
|
||||
::mediapipe::Status NormalizedLandmarkListToRect(
|
||||
const NormalizedLandmarkList& landmarks,
|
||||
const std::pair<int, int>& image_size, NormalizedRect* rect) {
|
||||
const float rotation = ComputeRotation(landmarks, image_size);
|
||||
const float reverse_angle = NormalizeRadians(-rotation);
|
||||
|
||||
// Find boundaries of landmarks.
|
||||
float max_x = std::numeric_limits<float>::min();
|
||||
float max_y = std::numeric_limits<float>::min();
|
||||
float min_x = std::numeric_limits<float>::max();
|
||||
float min_y = std::numeric_limits<float>::max();
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
max_x = std::max(max_x, landmarks.landmark(i).x());
|
||||
max_y = std::max(max_y, landmarks.landmark(i).y());
|
||||
min_x = std::min(min_x, landmarks.landmark(i).x());
|
||||
min_y = std::min(min_y, landmarks.landmark(i).y());
|
||||
}
|
||||
const float axis_aligned_center_x = (max_x + min_x) / 2.f;
|
||||
const float axis_aligned_center_y = (max_y + min_y) / 2.f;
|
||||
|
||||
// Find boundaries of rotated landmarks.
|
||||
max_x = std::numeric_limits<float>::min();
|
||||
max_y = std::numeric_limits<float>::min();
|
||||
min_x = std::numeric_limits<float>::max();
|
||||
min_y = std::numeric_limits<float>::max();
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const float original_x =
|
||||
(landmarks.landmark(i).x() - axis_aligned_center_x) * image_size.first;
|
||||
const float original_y =
|
||||
(landmarks.landmark(i).y() - axis_aligned_center_y) * image_size.second;
|
||||
|
||||
const float projected_x = original_x * std::cos(reverse_angle) -
|
||||
original_y * std::sin(reverse_angle);
|
||||
const float projected_y = original_x * std::sin(reverse_angle) +
|
||||
original_y * std::cos(reverse_angle);
|
||||
|
||||
max_x = std::max(max_x, projected_x);
|
||||
max_y = std::max(max_y, projected_y);
|
||||
min_x = std::min(min_x, projected_x);
|
||||
min_y = std::min(min_y, projected_y);
|
||||
}
|
||||
const float projected_center_x = (max_x + min_x) / 2.f;
|
||||
const float projected_center_y = (max_y + min_y) / 2.f;
|
||||
|
||||
const float center_x = projected_center_x * std::cos(rotation) -
|
||||
projected_center_y * std::sin(rotation) +
|
||||
image_size.first * axis_aligned_center_x;
|
||||
const float center_y = projected_center_x * std::sin(rotation) +
|
||||
projected_center_y * std::cos(rotation) +
|
||||
image_size.second * axis_aligned_center_y;
|
||||
const float width = (max_x - min_x) / image_size.first;
|
||||
const float height = (max_y - min_y) / image_size.second;
|
||||
|
||||
rect->set_x_center(center_x / image_size.first);
|
||||
rect->set_y_center(center_y / image_size.second);
|
||||
rect->set_width(width);
|
||||
rect->set_height(height);
|
||||
rect->set_rotation(rotation);
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator that converts subset of hand landmarks to a bounding box
|
||||
// NormalizedRect. The rotation angle of the bounding box is computed based on
|
||||
// 1) the wrist joint and 2) the average of PIP joints of index finger, middle
|
||||
// finger and ring finger. After rotation, the vector from the wrist to the mean
|
||||
// of PIP joints is expected to be vertical with wrist at the bottom and the
|
||||
// mean of PIP joints at the top.
|
||||
class HandLandmarksToRectCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
cc->Outputs().Tag(kNormRectTag).Set<NormalizedRect>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
|
||||
|
||||
std::pair<int, int> image_size =
|
||||
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
const auto& landmarks =
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
auto output_rect = absl::make_unique<NormalizedRect>();
|
||||
MP_RETURN_IF_ERROR(
|
||||
NormalizedLandmarkListToRect(landmarks, image_size, output_rect.get()));
|
||||
cc->Outputs()
|
||||
.Tag(kNormRectTag)
|
||||
.Add(output_rect.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(HandLandmarksToRectCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
@@ -88,7 +88,7 @@ mediapipe_simple_subgraph(
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/calculators:hand_detections_to_rects_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/calculators:hand_landmarks_to_rect_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -196,7 +196,7 @@ mediapipe_simple_subgraph(
|
||||
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/calculators:hand_detections_to_rects_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/calculators:hand_landmarks_to_rect_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -124,7 +124,7 @@ node {
|
||||
output_stream: "FLAG:hand_presence"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ThresholdingCalculatorOptions] {
|
||||
threshold: 0.1
|
||||
threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -195,21 +195,14 @@ node {
|
||||
}
|
||||
}
|
||||
|
||||
# Converts hand landmarks to a detection that tightly encloses all landmarks.
|
||||
node {
|
||||
calculator: "LandmarksToDetectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:partial_landmarks"
|
||||
output_stream: "DETECTION:hand_detection"
|
||||
}
|
||||
|
||||
# Converts the hand detection into a rectangle (normalized by image size)
|
||||
# Converts the hand landmarks into a rectangle (normalized by image size)
|
||||
# that encloses the hand. The calculator uses a subset of all hand landmarks
|
||||
# extracted from SplitNormalizedLandmarkListCalculator above to
|
||||
# calculate the bounding box and the rotation of the output rectangle. Please
|
||||
# see the comments in the calculator for more detail.
|
||||
node {
|
||||
calculator: "HandDetectionsToRectsCalculator"
|
||||
input_stream: "DETECTION:hand_detection"
|
||||
calculator: "HandLandmarksToRectCalculator"
|
||||
input_stream: "NORM_LANDMARKS:partial_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
}
|
||||
@@ -224,8 +217,8 @@ node {
|
||||
output_stream: "hand_rect_for_next_frame"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.1
|
||||
scale_y: 2.1
|
||||
scale_x: 2.0
|
||||
scale_y: 2.0
|
||||
shift_y: -0.1
|
||||
square_long: true
|
||||
}
|
||||
|
||||
@@ -128,7 +128,7 @@ node {
|
||||
output_stream: "FLAG:hand_presence"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ThresholdingCalculatorOptions] {
|
||||
threshold: 0.1
|
||||
threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -199,18 +199,14 @@ node {
|
||||
}
|
||||
}
|
||||
|
||||
# Converts hand landmarks to a detection that tightly encloses all landmarks.
|
||||
# Converts the hand landmarks into a rectangle (normalized by image size)
|
||||
# that encloses the hand. The calculator uses a subset of all hand landmarks
|
||||
# extracted from SplitNormalizedLandmarkListCalculator above to
|
||||
# calculate the bounding box and the rotation of the output rectangle. Please
|
||||
# see the comments in the calculator for more detail.
|
||||
node {
|
||||
calculator: "LandmarksToDetectionCalculator"
|
||||
calculator: "HandLandmarksToRectCalculator"
|
||||
input_stream: "NORM_LANDMARKS:partial_landmarks"
|
||||
output_stream: "DETECTION:hand_detection"
|
||||
}
|
||||
|
||||
# Converts the hand detection into a rectangle (normalized by image size)
|
||||
# that encloses the hand.
|
||||
node {
|
||||
calculator: "HandDetectionsToRectsCalculator"
|
||||
input_stream: "DETECTION:hand_detection"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
}
|
||||
@@ -225,8 +221,8 @@ node {
|
||||
output_stream: "hand_rect_for_next_frame"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.1
|
||||
scale_y: 2.1
|
||||
scale_x: 2.0
|
||||
scale_y: 2.0
|
||||
shift_y: -0.1
|
||||
square_long: true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load(
|
||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
cc_library(
|
||||
name = "instant_motion_tracking_deps",
|
||||
deps = [
|
||||
"//mediapipe/graphs/instant_motion_tracking/calculators:matrices_manager_calculator",
|
||||
"//mediapipe/graphs/instant_motion_tracking/calculators:sticker_manager_calculator",
|
||||
"//mediapipe/graphs/instant_motion_tracking/subgraphs:region_tracking",
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:gl_animation_overlay_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_binary_graph(
|
||||
name = "instant_motion_tracking_binary_graph",
|
||||
graph = "instant_motion_tracking.pbtxt",
|
||||
output_name = "instant_motion_tracking.binarypb",
|
||||
deps = [":instant_motion_tracking_deps"],
|
||||
)
|
||||
@@ -0,0 +1,86 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
exports_files(["LICENSE"])
|
||||
|
||||
proto_library(
|
||||
name = "sticker_buffer_proto",
|
||||
srcs = [
|
||||
"sticker_buffer.proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "sticker_buffer_cc_proto",
|
||||
srcs = [
|
||||
"sticker_buffer.proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":sticker_buffer_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "sticker_manager_calculator",
|
||||
srcs = ["sticker_manager_calculator.cc"],
|
||||
hdrs = ["transformations.h"],
|
||||
deps = [
|
||||
":sticker_buffer_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "matrices_manager_calculator",
|
||||
srcs = ["matrices_manager_calculator.cc"],
|
||||
hdrs = ["transformations.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:box",
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:model_matrix_cc_proto",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tracked_anchor_manager_calculator",
|
||||
srcs = ["tracked_anchor_manager_calculator.cc"],
|
||||
hdrs = ["transformations.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util/tracking:box_tracker_cc_proto",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -0,0 +1,394 @@
|
||||
// Copyright 2020 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
|
||||
#include "Eigen/Dense"
|
||||
#include "Eigen/src/Core/util/Constants.h"
|
||||
#include "Eigen/src/Geometry/Quaternion.h"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "absl/strings/str_join.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/graphs/instant_motion_tracking/calculators/transformations.h"
|
||||
#include "mediapipe/graphs/object_detection_3d/calculators/box.h"
|
||||
#include "mediapipe/graphs/object_detection_3d/calculators/model_matrix.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
using Matrix4fCM = Eigen::Matrix<float, 4, 4, Eigen::ColMajor>;
|
||||
using Vector3f = Eigen::Vector3f;
|
||||
using Matrix3f = Eigen::Matrix3f;
|
||||
using DiagonalMatrix3f = Eigen::DiagonalMatrix<float, 3>;
|
||||
constexpr char kAnchorsTag[] = "ANCHORS";
|
||||
constexpr char kIMUMatrixTag[] = "IMU_ROTATION";
|
||||
constexpr char kUserRotationsTag[] = "USER_ROTATIONS";
|
||||
constexpr char kUserScalingsTag[] = "USER_SCALINGS";
|
||||
constexpr char kRendersTag[] = "RENDER_DATA";
|
||||
constexpr char kGifAspectRatioTag[] = "GIF_ASPECT_RATIO";
|
||||
constexpr char kFOVSidePacketTag[] = "FOV";
|
||||
constexpr char kAspectRatioSidePacketTag[] = "ASPECT_RATIO";
|
||||
// initial Z value (-10 is center point in visual range for OpenGL render)
|
||||
constexpr float kInitialZ = -10.0f;
|
||||
} // namespace
|
||||
|
||||
// Intermediary for rotation and translation data to model matrix usable by
|
||||
// gl_animation_overlay_calculator. For information on the construction of
|
||||
// OpenGL objects and transformations (including a breakdown of model matrices),
|
||||
// please visit: https://open.gl/transformations
|
||||
//
|
||||
// Input Side Packets:
|
||||
// FOV - Vertical field of view for device [REQUIRED - Defines perspective
|
||||
// matrix] ASPECT_RATIO - Aspect ratio of device [REQUIRED - Defines
|
||||
// perspective matrix]
|
||||
//
|
||||
// Input streams:
|
||||
// ANCHORS - Anchor data with x,y,z coordinates (x,y are in [0.0-1.0] range for
|
||||
// position on the device screen, while z is the scaling factor that changes
|
||||
// in proportion to the distance from the tracked region) [REQUIRED]
|
||||
// IMU_ROTATION - float[9] of row-major device rotation matrix [REQUIRED]
|
||||
// USER_ROTATIONS - UserRotations with corresponding radians of rotation
|
||||
// [REQUIRED]
|
||||
// USER_SCALINGS - UserScalings with corresponding scale factor [REQUIRED]
|
||||
// GIF_ASPECT_RATIO - Aspect ratio of GIF image used to dynamically scale
|
||||
// GIF asset defined as width / height [OPTIONAL]
|
||||
// Output:
|
||||
// MATRICES - TimedModelMatrixProtoList of each object type to render
|
||||
// [REQUIRED]
|
||||
//
|
||||
// Example config:
|
||||
// node{
|
||||
// calculator: "MatricesManagerCalculator"
|
||||
// input_stream: "ANCHORS:tracked_scaled_anchor_data"
|
||||
// input_stream: "IMU_ROTATION:imu_rotation_matrix"
|
||||
// input_stream: "USER_ROTATIONS:user_rotation_data"
|
||||
// input_stream: "USER_SCALINGS:user_scaling_data"
|
||||
// input_stream: "GIF_ASPECT_RATIO:gif_aspect_ratio"
|
||||
// output_stream: "MATRICES:0:first_render_matrices"
|
||||
// output_stream: "MATRICES:1:second_render_matrices" [unbounded input size]
|
||||
// input_side_packet: "FOV:vertical_fov_radians"
|
||||
// input_side_packet: "ASPECT_RATIO:aspect_ratio"
|
||||
// }
|
||||
|
||||
class MatricesManagerCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Device properties that will be preset by side packets
|
||||
float vertical_fov_radians_ = 0.0f;
|
||||
float aspect_ratio_ = 0.0f;
|
||||
float gif_aspect_ratio_ = 1.0f;
|
||||
|
||||
const Matrix3f GenerateUserRotationMatrix(const float rotation_radians) const;
|
||||
const Matrix4fCM GenerateEigenModelMatrix(
|
||||
const Vector3f& translation_vector,
|
||||
const Matrix3f& rotation_submatrix) const;
|
||||
const Vector3f GenerateAnchorVector(const Anchor& tracked_anchor) const;
|
||||
DiagonalMatrix3f GetDefaultRenderScaleDiagonal(
|
||||
const int render_id, const float user_scale_factor,
|
||||
const float gif_aspect_ratio) const;
|
||||
|
||||
// Returns a user scaling increment associated with the sticker_id
|
||||
// TODO: Adjust lookup function if total number of stickers is uncapped to
|
||||
// improve performance
|
||||
const float GetUserScaler(const std::vector<UserScaling>& scalings,
|
||||
const int sticker_id) const {
|
||||
for (const UserScaling& user_scaling : scalings) {
|
||||
if (user_scaling.sticker_id == sticker_id) {
|
||||
return user_scaling.scale_factor;
|
||||
}
|
||||
}
|
||||
LOG(WARNING) << "Cannot find sticker_id: " << sticker_id
|
||||
<< ", returning 1.0f scaling";
|
||||
return 1.0f;
|
||||
}
|
||||
|
||||
// Returns a user rotation in radians associated with the sticker_id
|
||||
const float GetUserRotation(const std::vector<UserRotation>& rotations,
|
||||
const int sticker_id) {
|
||||
for (const UserRotation& rotation : rotations) {
|
||||
if (rotation.sticker_id == sticker_id) {
|
||||
return rotation.rotation_radians;
|
||||
}
|
||||
}
|
||||
LOG(WARNING) << "Cannot find sticker_id: " << sticker_id
|
||||
<< ", returning 0.0f rotation";
|
||||
return 0.0f;
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(MatricesManagerCalculator);
|
||||
|
||||
::mediapipe::Status MatricesManagerCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kAnchorsTag) &&
|
||||
cc->Inputs().HasTag(kIMUMatrixTag) &&
|
||||
cc->Inputs().HasTag(kUserRotationsTag) &&
|
||||
cc->Inputs().HasTag(kUserScalingsTag) &&
|
||||
cc->InputSidePackets().HasTag(kFOVSidePacketTag) &&
|
||||
cc->InputSidePackets().HasTag(kAspectRatioSidePacketTag));
|
||||
|
||||
cc->Inputs().Tag(kAnchorsTag).Set<std::vector<Anchor>>();
|
||||
cc->Inputs().Tag(kIMUMatrixTag).Set<float[]>();
|
||||
cc->Inputs().Tag(kUserScalingsTag).Set<std::vector<UserScaling>>();
|
||||
cc->Inputs().Tag(kUserRotationsTag).Set<std::vector<UserRotation>>();
|
||||
cc->Inputs().Tag(kRendersTag).Set<std::vector<int>>();
|
||||
if (cc->Inputs().HasTag(kGifAspectRatioTag)) {
|
||||
cc->Inputs().Tag(kGifAspectRatioTag).Set<float>();
|
||||
}
|
||||
|
||||
for (CollectionItemId id = cc->Outputs().BeginId("MATRICES");
|
||||
id < cc->Outputs().EndId("MATRICES"); ++id) {
|
||||
cc->Outputs().Get(id).Set<mediapipe::TimedModelMatrixProtoList>();
|
||||
}
|
||||
cc->InputSidePackets().Tag(kFOVSidePacketTag).Set<float>();
|
||||
cc->InputSidePackets().Tag(kAspectRatioSidePacketTag).Set<float>();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status MatricesManagerCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
// Set device properties from side packets
|
||||
vertical_fov_radians_ =
|
||||
cc->InputSidePackets().Tag(kFOVSidePacketTag).Get<float>();
|
||||
aspect_ratio_ =
|
||||
cc->InputSidePackets().Tag(kAspectRatioSidePacketTag).Get<float>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status MatricesManagerCalculator::Process(CalculatorContext* cc) {
|
||||
// Define each object's model matrices
|
||||
auto asset_matrices_gif =
|
||||
std::make_unique<mediapipe::TimedModelMatrixProtoList>();
|
||||
auto asset_matrices_1 =
|
||||
std::make_unique<mediapipe::TimedModelMatrixProtoList>();
|
||||
// Clear all model matrices
|
||||
asset_matrices_gif->clear_model_matrix();
|
||||
asset_matrices_1->clear_model_matrix();
|
||||
|
||||
const std::vector<UserRotation> user_rotation_data =
|
||||
cc->Inputs().Tag(kUserRotationsTag).Get<std::vector<UserRotation>>();
|
||||
|
||||
const std::vector<UserScaling> user_scaling_data =
|
||||
cc->Inputs().Tag(kUserScalingsTag).Get<std::vector<UserScaling>>();
|
||||
|
||||
const std::vector<int> render_data =
|
||||
cc->Inputs().Tag(kRendersTag).Get<std::vector<int>>();
|
||||
|
||||
const std::vector<Anchor> anchor_data =
|
||||
cc->Inputs().Tag(kAnchorsTag).Get<std::vector<Anchor>>();
|
||||
if (cc->Inputs().HasTag(kGifAspectRatioTag) &&
|
||||
!cc->Inputs().Tag(kGifAspectRatioTag).IsEmpty()) {
|
||||
gif_aspect_ratio_ = cc->Inputs().Tag(kGifAspectRatioTag).Get<float>();
|
||||
}
|
||||
|
||||
// Device IMU rotation submatrix
|
||||
const auto imu_matrix = cc->Inputs().Tag(kIMUMatrixTag).Get<float[]>();
|
||||
Matrix3f imu_rotation_submatrix;
|
||||
int idx = 0;
|
||||
for (int x = 0; x < 3; ++x) {
|
||||
for (int y = 0; y < 3; ++y) {
|
||||
// Input matrix is row-major matrix, it must be reformatted to
|
||||
// column-major via transpose procedure
|
||||
imu_rotation_submatrix(y, x) = imu_matrix[idx++];
|
||||
}
|
||||
}
|
||||
|
||||
int render_idx = 0;
|
||||
for (const Anchor& anchor : anchor_data) {
|
||||
const int id = anchor.sticker_id;
|
||||
mediapipe::TimedModelMatrixProto* model_matrix;
|
||||
// Add model matrix to matrices list for defined object render ID
|
||||
if (render_data[render_idx] == 0) { // GIF
|
||||
model_matrix = asset_matrices_gif->add_model_matrix();
|
||||
} else { // Asset 3D
|
||||
if (render_data[render_idx] != 1) {
|
||||
LOG(ERROR) << "render id: " << render_data[render_idx]
|
||||
<< " is not supported. Fall back to using render_id = 1.";
|
||||
}
|
||||
model_matrix = asset_matrices_1->add_model_matrix();
|
||||
}
|
||||
|
||||
model_matrix->set_id(id);
|
||||
|
||||
// The user transformation data associated with this sticker must be defined
|
||||
const float user_rotation_radians = GetUserRotation(user_rotation_data, id);
|
||||
const float user_scale_factor = GetUserScaler(user_scaling_data, id);
|
||||
|
||||
// A vector representative of a user's sticker rotation transformation can
|
||||
// be created
|
||||
const Matrix3f user_rotation_submatrix =
|
||||
GenerateUserRotationMatrix(user_rotation_radians);
|
||||
// Next, the diagonal representative of the combined scaling data
|
||||
const DiagonalMatrix3f scaling_diagonal = GetDefaultRenderScaleDiagonal(
|
||||
render_data[render_idx], user_scale_factor, gif_aspect_ratio_);
|
||||
// Increment to next render id from vector
|
||||
render_idx++;
|
||||
|
||||
// The user transformation data can be concatenated into a final rotation
|
||||
// submatrix with the device IMU rotational data
|
||||
const Matrix3f user_transformed_rotation_submatrix =
|
||||
imu_rotation_submatrix * user_rotation_submatrix * scaling_diagonal;
|
||||
|
||||
// A vector representative of the translation of the object in OpenGL
|
||||
// coordinate space must be generated
|
||||
const Vector3f translation_vector = GenerateAnchorVector(anchor);
|
||||
|
||||
// Concatenate all model matrix data
|
||||
const Matrix4fCM final_model_matrix = GenerateEigenModelMatrix(
|
||||
translation_vector, user_transformed_rotation_submatrix);
|
||||
|
||||
// The generated model matrix must be mapped to TimedModelMatrixProto
|
||||
// (col-wise)
|
||||
for (int x = 0; x < final_model_matrix.rows(); ++x) {
|
||||
for (int y = 0; y < final_model_matrix.cols(); ++y) {
|
||||
model_matrix->add_matrix_entries(final_model_matrix(x, y));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Output all individual render matrices
|
||||
// TODO: Perform depth ordering with gl_animation_overlay_calculator to render
|
||||
// objects in order by depth to allow occlusion.
|
||||
cc->Outputs()
|
||||
.Get(cc->Outputs().GetId("MATRICES", 0))
|
||||
.Add(asset_matrices_gif.release(), cc->InputTimestamp());
|
||||
cc->Outputs()
|
||||
.Get(cc->Outputs().GetId("MATRICES", 1))
|
||||
.Add(asset_matrices_1.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
// Using a specified rotation value in radians, generate a rotation matrix for
|
||||
// use with base rotation submatrix
|
||||
const Matrix3f MatricesManagerCalculator::GenerateUserRotationMatrix(
|
||||
const float rotation_radians) const {
|
||||
Eigen::Matrix3f user_rotation_submatrix;
|
||||
user_rotation_submatrix =
|
||||
// The rotation in radians must be inverted to rotate the object
|
||||
// with the direction of finger movement from the user (system dependent)
|
||||
Eigen::AngleAxisf(-rotation_radians, Eigen::Vector3f::UnitY()) *
|
||||
Eigen::AngleAxisf(0.0f, Eigen::Vector3f::UnitZ()) *
|
||||
// Model orientations all assume z-axis is up, but we need y-axis upwards,
|
||||
// therefore, a +(M_PI * 0.5f) transformation must be applied
|
||||
// TODO: Bring default rotations, translations, and scalings into
|
||||
// independent sticker configuration
|
||||
Eigen::AngleAxisf(M_PI * 0.5f, Eigen::Vector3f::UnitX());
|
||||
// Matrix must be transposed due to the method of submatrix generation in
|
||||
// Eigen
|
||||
return user_rotation_submatrix.transpose();
|
||||
}
|
||||
|
||||
// TODO: Investigate possible differences in warping of tracking speed across
|
||||
// screen Using the sticker anchor data, a translation vector can be generated
|
||||
// in OpenGL coordinate space
|
||||
const Vector3f MatricesManagerCalculator::GenerateAnchorVector(
|
||||
const Anchor& tracked_anchor) const {
|
||||
// Using an initial z-value in OpenGL space, generate a new base z-axis value
|
||||
// to mimic scaling by distance.
|
||||
const float z = kInitialZ * tracked_anchor.z;
|
||||
|
||||
// Using triangle geometry, the minimum for a y-coordinate that will appear in
|
||||
// the view field for the given z value above can be found.
|
||||
const float y_half_range = z * (tan(vertical_fov_radians_ * 0.5f));
|
||||
|
||||
// The aspect ratio of the device and y_minimum calculated above can be used
|
||||
// to find the minimum value for x that will appear in the view field of the
|
||||
// device screen.
|
||||
const float x_half_range = y_half_range * aspect_ratio_;
|
||||
|
||||
// Given the minimum bounds of the screen in OpenGL space, the tracked anchor
|
||||
// coordinates can be converted to OpenGL coordinate space.
|
||||
//
|
||||
// (i.e: X and Y will be converted from [0.0-1.0] space to [x_minimum,
|
||||
// -x_minimum] space and [y_minimum, -y_minimum] space respectively)
|
||||
const float x = (-2.0f * tracked_anchor.x * x_half_range) + x_half_range;
|
||||
const float y = (-2.0f * tracked_anchor.y * y_half_range) + y_half_range;
|
||||
|
||||
// A translation transformation vector can be generated via Eigen
|
||||
const Vector3f t_vector(x, y, z);
|
||||
return t_vector;
|
||||
}
|
||||
|
||||
// Generates a model matrix via Eigen with appropriate transformations
|
||||
const Matrix4fCM MatricesManagerCalculator::GenerateEigenModelMatrix(
|
||||
const Vector3f& translation_vector,
|
||||
const Matrix3f& rotation_submatrix) const {
|
||||
// Define basic empty model matrix
|
||||
Matrix4fCM mvp_matrix;
|
||||
|
||||
// Set the translation vector
|
||||
mvp_matrix.topRightCorner<3, 1>() = translation_vector;
|
||||
|
||||
// Set the rotation submatrix
|
||||
mvp_matrix.topLeftCorner<3, 3>() = rotation_submatrix;
|
||||
|
||||
// Set trailing 1.0 required by OpenGL to define coordinate space
|
||||
mvp_matrix(3, 3) = 1.0f;
|
||||
|
||||
return mvp_matrix;
|
||||
}
|
||||
|
||||
// This returns a scaling matrix to alter the projection matrix for
|
||||
// the specified render id in order to ensure all objects render at a similar
|
||||
// size in the view screen upon initial placement
|
||||
DiagonalMatrix3f MatricesManagerCalculator::GetDefaultRenderScaleDiagonal(
|
||||
const int render_id, const float user_scale_factor,
|
||||
const float gif_aspect_ratio) const {
|
||||
float scale_preset = 1.0f;
|
||||
float x_scalar = 1.0f;
|
||||
float y_scalar = 1.0f;
|
||||
|
||||
switch (render_id) {
|
||||
case 0: { // GIF
|
||||
// 160 is the scaling preset to make the GIF asset appear relatively
|
||||
// similar in size to all other assets
|
||||
scale_preset = 160.0f;
|
||||
if (gif_aspect_ratio >= 1.0f) {
|
||||
// GIF is wider horizontally (scale on x-axis)
|
||||
x_scalar = gif_aspect_ratio;
|
||||
} else {
|
||||
// GIF is wider vertically (scale on y-axis)
|
||||
y_scalar = 1.0f / gif_aspect_ratio;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 1: { // 3D asset
|
||||
// 5 is the scaling preset to make the 3D asset appear relatively
|
||||
// similar in size to all other assets
|
||||
scale_preset = 5.0f;
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
LOG(INFO) << "Unsupported render_id: " << render_id
|
||||
<< ", returning default render_scale";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
DiagonalMatrix3f scaling(scale_preset * user_scale_factor * x_scalar,
|
||||
scale_preset * user_scale_factor * y_scalar,
|
||||
scale_preset * user_scale_factor);
|
||||
return scaling;
|
||||
}
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2020 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
option java_package = "com.google.mediapipe.graphs.instantmotiontracking";
|
||||
option java_outer_classname = "StickerBufferProto";
|
||||
|
||||
message Sticker {
|
||||
optional int32 id = 1;
|
||||
optional float x = 2;
|
||||
optional float y = 3;
|
||||
optional float rotation = 4;
|
||||
optional float scale = 5;
|
||||
optional int32 render_id = 6;
|
||||
}
|
||||
|
||||
message StickerRoll {
|
||||
repeated Sticker sticker = 1;
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2020 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/graphs/instant_motion_tracking/calculators/sticker_buffer.pb.h"
|
||||
#include "mediapipe/graphs/instant_motion_tracking/calculators/transformations.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kProtoDataString[] = "PROTO";
|
||||
constexpr char kAnchorsTag[] = "ANCHORS";
|
||||
constexpr char kUserRotationsTag[] = "USER_ROTATIONS";
|
||||
constexpr char kUserScalingsTag[] = "USER_SCALINGS";
|
||||
constexpr char kRenderDescriptorsTag[] = "RENDER_DATA";
|
||||
|
||||
// This calculator takes in the sticker protobuffer data and parses each
|
||||
// individual sticker object into anchors, user rotations and scalings, in
|
||||
// addition to basic render data represented in integer form.
|
||||
//
|
||||
// Input:
|
||||
// PROTO - String of sticker data in appropriate protobuf format [REQUIRED]
|
||||
// Output:
|
||||
// ANCHORS - Anchors with initial normalized X,Y coordinates [REQUIRED]
|
||||
// USER_ROTATIONS - UserRotations with radians of rotation from user [REQUIRED]
|
||||
// USER_SCALINGS - UserScalings with increment of scaling from user [REQUIRED]
|
||||
// RENDER_DATA - Descriptors of which objects/animations to render for stickers
|
||||
// [REQUIRED]
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "StickerManagerCalculator"
|
||||
// input_stream: "PROTO:sticker_proto_string"
|
||||
// output_stream: "ANCHORS:initial_anchor_data"
|
||||
// output_stream: "USER_ROTATIONS:user_rotation_data"
|
||||
// output_stream: "USER_SCALINGS:user_scaling_data"
|
||||
// output_stream: "RENDER_DATA:sticker_render_data"
|
||||
// }
|
||||
|
||||
class StickerManagerCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kProtoDataString));
|
||||
RET_CHECK(cc->Outputs().HasTag(kAnchorsTag) &&
|
||||
cc->Outputs().HasTag(kUserRotationsTag) &&
|
||||
cc->Outputs().HasTag(kUserScalingsTag) &&
|
||||
cc->Outputs().HasTag(kRenderDescriptorsTag));
|
||||
|
||||
cc->Inputs().Tag(kProtoDataString).Set<std::string>();
|
||||
cc->Outputs().Tag(kAnchorsTag).Set<std::vector<Anchor>>();
|
||||
cc->Outputs().Tag(kUserRotationsTag).Set<std::vector<UserRotation>>();
|
||||
cc->Outputs().Tag(kUserScalingsTag).Set<std::vector<UserScaling>>();
|
||||
cc->Outputs().Tag(kRenderDescriptorsTag).Set<std::vector<int>>();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
std::string sticker_proto_string =
|
||||
cc->Inputs().Tag(kProtoDataString).Get<std::string>();
|
||||
|
||||
std::vector<Anchor> initial_anchor_data;
|
||||
std::vector<UserRotation> user_rotation_data;
|
||||
std::vector<UserScaling> user_scaling_data;
|
||||
std::vector<int> render_data;
|
||||
|
||||
::mediapipe::StickerRoll sticker_roll;
|
||||
bool parse_success = sticker_roll.ParseFromString(sticker_proto_string);
|
||||
|
||||
// Ensure parsing was a success
|
||||
RET_CHECK(parse_success) << "Error parsing sticker protobuf data";
|
||||
|
||||
for (int i = 0; i < sticker_roll.sticker().size(); ++i) {
|
||||
// Declare empty structures for sticker data
|
||||
Anchor initial_anchor;
|
||||
UserRotation user_rotation;
|
||||
UserScaling user_scaling;
|
||||
// Get individual Sticker object as defined by Protobuffer
|
||||
::mediapipe::Sticker sticker = sticker_roll.sticker(i);
|
||||
// Set individual data structure ids to associate with this sticker
|
||||
initial_anchor.sticker_id = sticker.id();
|
||||
user_rotation.sticker_id = sticker.id();
|
||||
user_scaling.sticker_id = sticker.id();
|
||||
initial_anchor.x = sticker.x();
|
||||
initial_anchor.y = sticker.y();
|
||||
initial_anchor.z = 1.0f; // default to 1.0 in normalized 3d space
|
||||
user_rotation.rotation_radians = sticker.rotation();
|
||||
user_scaling.scale_factor = sticker.scale();
|
||||
const int render_id = sticker.render_id();
|
||||
// Set all vector data with sticker attributes
|
||||
initial_anchor_data.emplace_back(initial_anchor);
|
||||
user_rotation_data.emplace_back(user_rotation);
|
||||
user_scaling_data.emplace_back(user_scaling);
|
||||
render_data.emplace_back(render_id);
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag(kAnchorsTag)) {
|
||||
cc->Outputs()
|
||||
.Tag(kAnchorsTag)
|
||||
.AddPacket(MakePacket<std::vector<Anchor>>(initial_anchor_data)
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
if (cc->Outputs().HasTag(kUserRotationsTag)) {
|
||||
cc->Outputs()
|
||||
.Tag(kUserRotationsTag)
|
||||
.AddPacket(MakePacket<std::vector<UserRotation>>(user_rotation_data)
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
if (cc->Outputs().HasTag(kUserScalingsTag)) {
|
||||
cc->Outputs()
|
||||
.Tag(kUserScalingsTag)
|
||||
.AddPacket(MakePacket<std::vector<UserScaling>>(user_scaling_data)
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
if (cc->Outputs().HasTag(kRenderDescriptorsTag)) {
|
||||
cc->Outputs()
|
||||
.Tag(kRenderDescriptorsTag)
|
||||
.AddPacket(MakePacket<std::vector<int>>(render_data)
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(StickerManagerCalculator);
|
||||
} // namespace mediapipe
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
// Copyright 2020 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/graphs/instant_motion_tracking/calculators/transformations.h"
|
||||
#include "mediapipe/util/tracking/box_tracker.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kSentinelTag[] = "SENTINEL";
|
||||
constexpr char kAnchorsTag[] = "ANCHORS";
|
||||
constexpr char kBoxesInputTag[] = "BOXES";
|
||||
constexpr char kBoxesOutputTag[] = "START_POS";
|
||||
constexpr char kCancelTag[] = "CANCEL_ID";
|
||||
// TODO: Find optimal Height/Width (0.1-0.3)
|
||||
constexpr float kBoxEdgeSize =
|
||||
0.2f; // Used to establish tracking box dimensions
|
||||
constexpr float kUsToMs =
|
||||
1000.0f; // Used to convert from microseconds to millis
|
||||
|
||||
// This calculator manages the regions being tracked for each individual sticker
|
||||
// and adjusts the regions being tracked if a change is detected in a sticker's
|
||||
// initial anchor placement. Regions being tracked that have no associated
|
||||
// sticker will be automatically removed upon the next iteration of the graph to
|
||||
// optimize performance and remove all sticker artifacts
|
||||
//
|
||||
// Input:
|
||||
// SENTINEL - ID of sticker which has an anchor that must be reset (-1 when no
|
||||
// anchor must be reset) [REQUIRED]
|
||||
// ANCHORS - Initial anchor data (tracks changes and where to re/position)
|
||||
// [REQUIRED] BOXES - Used in cycle, boxes being tracked meant to update
|
||||
// positions [OPTIONAL
|
||||
// - provided by subgraph]
|
||||
// Output:
|
||||
// START_POS - Positions of boxes being tracked (can be overwritten with ID)
|
||||
// [REQUIRED] CANCEL_ID - Single integer ID of tracking box to remove from
|
||||
// tracker subgraph [OPTIONAL] ANCHORS - Updated set of anchors with tracked
|
||||
// and normalized X,Y,Z [REQUIRED]
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "TrackedAnchorManagerCalculator"
|
||||
// input_stream: "SENTINEL:sticker_sentinel"
|
||||
// input_stream: "ANCHORS:initial_anchor_data"
|
||||
// input_stream: "BOXES:boxes"
|
||||
// input_stream_info: {
|
||||
// tag_index: 'BOXES'
|
||||
// back_edge: true
|
||||
// }
|
||||
// output_stream: "START_POS:start_pos"
|
||||
// output_stream: "CANCEL_ID:cancel_object_id"
|
||||
// output_stream: "ANCHORS:tracked_scaled_anchor_data"
|
||||
// }
|
||||
|
||||
class TrackedAnchorManagerCalculator : public CalculatorBase {
|
||||
private:
|
||||
// Previous graph iteration anchor data
|
||||
std::vector<Anchor> previous_anchor_data_;
|
||||
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kAnchorsTag) &&
|
||||
cc->Inputs().HasTag(kSentinelTag));
|
||||
RET_CHECK(cc->Outputs().HasTag(kAnchorsTag) &&
|
||||
cc->Outputs().HasTag(kBoxesOutputTag));
|
||||
|
||||
cc->Inputs().Tag(kAnchorsTag).Set<std::vector<Anchor>>();
|
||||
cc->Inputs().Tag(kSentinelTag).Set<int>();
|
||||
|
||||
if (cc->Inputs().HasTag(kBoxesInputTag)) {
|
||||
cc->Inputs().Tag(kBoxesInputTag).Set<mediapipe::TimedBoxProtoList>();
|
||||
}
|
||||
|
||||
cc->Outputs().Tag(kAnchorsTag).Set<std::vector<Anchor>>();
|
||||
cc->Outputs().Tag(kBoxesOutputTag).Set<mediapipe::TimedBoxProtoList>();
|
||||
|
||||
if (cc->Outputs().HasTag(kCancelTag)) {
|
||||
cc->Outputs().Tag(kCancelTag).Set<int>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(TrackedAnchorManagerCalculator);
|
||||
|
||||
::mediapipe::Status TrackedAnchorManagerCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
mediapipe::Timestamp timestamp = cc->InputTimestamp();
|
||||
const int sticker_sentinel = cc->Inputs().Tag(kSentinelTag).Get<int>();
|
||||
std::vector<Anchor> current_anchor_data =
|
||||
cc->Inputs().Tag(kAnchorsTag).Get<std::vector<Anchor>>();
|
||||
auto pos_boxes = absl::make_unique<mediapipe::TimedBoxProtoList>();
|
||||
std::vector<Anchor> tracked_scaled_anchor_data;
|
||||
|
||||
// Delete any boxes being tracked without an associated anchor
|
||||
for (const mediapipe::TimedBoxProto& box :
|
||||
cc->Inputs()
|
||||
.Tag(kBoxesInputTag)
|
||||
.Get<mediapipe::TimedBoxProtoList>()
|
||||
.box()) {
|
||||
bool anchor_exists = false;
|
||||
for (Anchor anchor : current_anchor_data) {
|
||||
if (box.id() == anchor.sticker_id) {
|
||||
anchor_exists = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!anchor_exists) {
|
||||
cc->Outputs()
|
||||
.Tag(kCancelTag)
|
||||
.AddPacket(MakePacket<int>(box.id()).At(timestamp++));
|
||||
}
|
||||
}
|
||||
|
||||
// Perform tracking or updating for each anchor position
|
||||
for (const Anchor& anchor : current_anchor_data) {
|
||||
Anchor output_anchor = anchor;
|
||||
// Check if anchor position is being reset by user in this graph iteration
|
||||
if (sticker_sentinel == anchor.sticker_id) {
|
||||
// Delete associated tracking box
|
||||
// TODO: BoxTrackingSubgraph should accept vector to avoid breaking
|
||||
// timestamp rules
|
||||
cc->Outputs()
|
||||
.Tag(kCancelTag)
|
||||
.AddPacket(MakePacket<int>(anchor.sticker_id).At(timestamp++));
|
||||
// Add a tracking box
|
||||
mediapipe::TimedBoxProto* box = pos_boxes->add_box();
|
||||
box->set_left(anchor.x - kBoxEdgeSize * 0.5f);
|
||||
box->set_right(anchor.x + kBoxEdgeSize * 0.5f);
|
||||
box->set_top(anchor.y - kBoxEdgeSize * 0.5f);
|
||||
box->set_bottom(anchor.y + kBoxEdgeSize * 0.5f);
|
||||
box->set_id(anchor.sticker_id);
|
||||
box->set_time_msec((timestamp++).Microseconds() / kUsToMs);
|
||||
// Default value for normalized z (scale factor)
|
||||
output_anchor.z = 1.0f;
|
||||
} else {
|
||||
// Anchor position was not reset by user
|
||||
// Attempt to update anchor position from tracking subgraph
|
||||
// (TimedBoxProto)
|
||||
bool updated_from_tracker = false;
|
||||
const mediapipe::TimedBoxProtoList box_list =
|
||||
cc->Inputs().Tag(kBoxesInputTag).Get<mediapipe::TimedBoxProtoList>();
|
||||
for (const auto& box : box_list.box()) {
|
||||
if (box.id() == anchor.sticker_id) {
|
||||
// Get center x normalized coordinate [0.0-1.0]
|
||||
output_anchor.x = (box.left() + box.right()) * 0.5f;
|
||||
// Get center y normalized coordinate [0.0-1.0]
|
||||
output_anchor.y = (box.top() + box.bottom()) * 0.5f;
|
||||
// Get center z coordinate [z starts at normalized 1.0 and scales
|
||||
// inversely with box-width]
|
||||
// TODO: Look into issues with uniform scaling on x-axis and y-axis
|
||||
output_anchor.z = kBoxEdgeSize / (box.right() - box.left());
|
||||
updated_from_tracker = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// If anchor position was not updated from tracker, create new tracking
|
||||
// box at last recorded anchor coordinates. This will allow all current
|
||||
// stickers to be tracked at approximately last location even if
|
||||
// re-acquisitioning in the BoxTrackingSubgraph encounters errors
|
||||
if (!updated_from_tracker) {
|
||||
for (const Anchor& prev_anchor : previous_anchor_data_) {
|
||||
if (anchor.sticker_id == prev_anchor.sticker_id) {
|
||||
mediapipe::TimedBoxProto* box = pos_boxes->add_box();
|
||||
box->set_left(prev_anchor.x - kBoxEdgeSize * 0.5f);
|
||||
box->set_right(prev_anchor.x + kBoxEdgeSize * 0.5f);
|
||||
box->set_top(prev_anchor.y - kBoxEdgeSize * 0.5f);
|
||||
box->set_bottom(prev_anchor.y + kBoxEdgeSize * 0.5f);
|
||||
box->set_id(prev_anchor.sticker_id);
|
||||
box->set_time_msec(cc->InputTimestamp().Microseconds() / kUsToMs);
|
||||
output_anchor = prev_anchor;
|
||||
// Default value for normalized z (scale factor)
|
||||
output_anchor.z = 1.0f;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
tracked_scaled_anchor_data.emplace_back(output_anchor);
|
||||
}
|
||||
// Set anchor data for next iteration
|
||||
previous_anchor_data_ = tracked_scaled_anchor_data;
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kAnchorsTag)
|
||||
.AddPacket(MakePacket<std::vector<Anchor>>(tracked_scaled_anchor_data)
|
||||
.At(cc->InputTimestamp()));
|
||||
cc->Outputs()
|
||||
.Tag(kBoxesOutputTag)
|
||||
.Add(pos_boxes.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright 2020 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_GRAPHS_INSTANT_MOTION_TRACKING_CALCULATORS_TRANSFORMATIONS_H_
|
||||
#define MEDIAPIPE_GRAPHS_INSTANT_MOTION_TRACKING_CALCULATORS_TRANSFORMATIONS_H_
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Radians by which to rotate the object (Provided by UI input)
|
||||
struct UserRotation {
|
||||
float rotation_radians;
|
||||
int sticker_id;
|
||||
};
|
||||
|
||||
// Scaling factor provided by the UI application end
|
||||
struct UserScaling {
|
||||
float scale_factor;
|
||||
int sticker_id;
|
||||
};
|
||||
|
||||
// The normalized anchor coordinates of a sticker
|
||||
struct Anchor {
|
||||
float x; // [0.0-1.0]
|
||||
float y; // [0.0-1.0]
|
||||
float z; // Centered around 1.0 [current_scale = z * initial_scale]
|
||||
int sticker_id;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_GRAPHS_INSTANT_MOTION_TRACKING_CALCULATORS_TRANSFORMATIONS_H_
|
||||
@@ -0,0 +1,80 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# MediaPipe graph that performs region tracking and 3d object (AR sticker) rendering.
|
||||
|
||||
# Images in/out of graph with sticker data and IMU information from device
|
||||
input_stream: "input_video"
|
||||
input_stream: "sticker_sentinel"
|
||||
input_stream: "sticker_proto_string"
|
||||
input_stream: "imu_rotation_matrix"
|
||||
input_stream: "gif_texture"
|
||||
input_stream: "gif_aspect_ratio"
|
||||
output_stream: "output_video"
|
||||
|
||||
# Converts sticker data into user data (rotations/scalings), render data, and
|
||||
# initial anchors.
|
||||
node {
|
||||
calculator: "StickerManagerCalculator"
|
||||
input_stream: "PROTO:sticker_proto_string"
|
||||
output_stream: "ANCHORS:initial_anchor_data"
|
||||
output_stream: "USER_ROTATIONS:user_rotation_data"
|
||||
output_stream: "USER_SCALINGS:user_scaling_data"
|
||||
output_stream: "RENDER_DATA:sticker_render_data"
|
||||
}
|
||||
|
||||
# Uses box tracking in order to create 'anchors' for associated 3d stickers.
|
||||
node {
|
||||
calculator: "RegionTrackingSubgraph"
|
||||
input_stream: "VIDEO:input_video"
|
||||
input_stream: "SENTINEL:sticker_sentinel"
|
||||
input_stream: "ANCHORS:initial_anchor_data"
|
||||
output_stream: "ANCHORS:tracked_anchor_data"
|
||||
}
|
||||
|
||||
# Concatenates all transformations to generate model matrices for the OpenGL
|
||||
# animation overlay calculator.
|
||||
node {
|
||||
calculator: "MatricesManagerCalculator"
|
||||
input_stream: "ANCHORS:tracked_anchor_data"
|
||||
input_stream: "IMU_ROTATION:imu_rotation_matrix"
|
||||
input_stream: "USER_ROTATIONS:user_rotation_data"
|
||||
input_stream: "USER_SCALINGS:user_scaling_data"
|
||||
input_stream: "RENDER_DATA:sticker_render_data"
|
||||
input_stream: "GIF_ASPECT_RATIO:gif_aspect_ratio"
|
||||
output_stream: "MATRICES:0:gif_matrices"
|
||||
output_stream: "MATRICES:1:asset_3d_matrices"
|
||||
input_side_packet: "FOV:vertical_fov_radians"
|
||||
input_side_packet: "ASPECT_RATIO:aspect_ratio"
|
||||
}
|
||||
|
||||
# Renders the final 3d stickers and overlays them on input image.
|
||||
node {
|
||||
calculator: "GlAnimationOverlayCalculator"
|
||||
input_stream: "VIDEO:input_video"
|
||||
input_stream: "MODEL_MATRICES:gif_matrices"
|
||||
input_stream: "TEXTURE:gif_texture"
|
||||
input_side_packet: "ANIMATION_ASSET:gif_asset_name"
|
||||
output_stream: "asset_gif_rendered"
|
||||
}
|
||||
|
||||
# Renders the final 3d stickers and overlays them on top of the input image.
|
||||
node {
|
||||
calculator: "GlAnimationOverlayCalculator"
|
||||
input_stream: "VIDEO:asset_gif_rendered"
|
||||
input_stream: "MODEL_MATRICES:asset_3d_matrices"
|
||||
input_side_packet: "TEXTURE:texture_3d"
|
||||
input_side_packet: "ANIMATION_ASSET:asset_3d"
|
||||
output_stream: "output_video"
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load(
|
||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "region_tracking",
|
||||
graph = "region_tracking.pbtxt",
|
||||
register_as = "RegionTrackingSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/graphs/instant_motion_tracking/calculators:tracked_anchor_manager_calculator",
|
||||
"//mediapipe/graphs/tracking/subgraphs:box_tracking_gpu",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# MediaPipe graph that performs region tracking on initial anchor positions
|
||||
# provided by the application
|
||||
|
||||
# Images in/out of graph with tracked and scaled normalized anchor data
|
||||
type: "RegionTrackingSubgraph"
|
||||
input_stream: "VIDEO:input_video"
|
||||
input_stream: "SENTINEL:sticker_sentinel"
|
||||
input_stream: "ANCHORS:initial_anchor_data"
|
||||
output_stream: "ANCHORS:tracked_scaled_anchor_data"
|
||||
|
||||
# Manages the anchors and tracking if user changes/adds/deletes anchors
|
||||
node {
|
||||
calculator: "TrackedAnchorManagerCalculator"
|
||||
input_stream: "SENTINEL:sticker_sentinel"
|
||||
input_stream: "ANCHORS:initial_anchor_data"
|
||||
input_stream: "BOXES:boxes"
|
||||
input_stream_info: {
|
||||
tag_index: 'BOXES'
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "START_POS:start_pos"
|
||||
output_stream: "CANCEL_ID:cancel_object_id"
|
||||
output_stream: "ANCHORS:tracked_scaled_anchor_data"
|
||||
}
|
||||
|
||||
# Subgraph performs anchor placement and tracking
|
||||
node {
|
||||
calculator: "BoxTrackingSubgraphGpu"
|
||||
input_stream: "VIDEO:input_video"
|
||||
input_stream: "BOXES:start_pos"
|
||||
input_stream: "CANCEL_ID:cancel_object_id"
|
||||
output_stream: "BOXES:boxes"
|
||||
}
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
@@ -30,6 +30,7 @@ cc_library(
|
||||
"//mediapipe/calculators/image:image_file_properties_calculator",
|
||||
"//mediapipe/calculators/image:opencv_encoded_image_to_image_frame_calculator",
|
||||
"//mediapipe/calculators/image:opencv_image_encoder_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/subgraphs:iris_and_depth_renderer_cpu",
|
||||
"//mediapipe/modules/face_landmark:face_landmark_front_cpu",
|
||||
"//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_cpu",
|
||||
@@ -42,6 +43,7 @@ cc_library(
|
||||
"//mediapipe/calculators/core:constant_side_packet_calculator",
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/subgraphs:iris_renderer_cpu",
|
||||
"//mediapipe/modules/face_landmark:face_landmark_front_cpu",
|
||||
"//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_cpu",
|
||||
@@ -56,6 +58,7 @@ cc_library(
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/video:opencv_video_decoder_calculator",
|
||||
"//mediapipe/calculators/video:opencv_video_encoder_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/subgraphs:iris_renderer_cpu",
|
||||
"//mediapipe/modules/face_landmark:face_landmark_front_cpu",
|
||||
"//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_cpu",
|
||||
@@ -68,6 +71,7 @@ cc_library(
|
||||
"//mediapipe/calculators/core:constant_side_packet_calculator",
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/subgraphs:iris_and_depth_renderer_gpu",
|
||||
"//mediapipe/modules/face_landmark:face_landmark_front_gpu",
|
||||
"//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_gpu",
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
proto_library(
|
||||
name = "iris_to_render_data_calculator_proto",
|
||||
@@ -90,3 +90,18 @@ cc_library(
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "update_face_landmarks_calculator",
|
||||
srcs = ["update_face_landmarks_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image_file_properties_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kFaceLandmarksTag[] = "FACE_LANDMARKS";
|
||||
constexpr char kNewEyeLandmarksTag[] = "NEW_EYE_LANDMARKS";
|
||||
constexpr char kUpdatedFaceLandmarksTag[] = "UPDATED_FACE_LANDMARKS";
|
||||
|
||||
constexpr int kNumFaceLandmarks = 468;
|
||||
// 71 landamrks for left eye and 71 landmarks for right eye.
|
||||
constexpr int kNumEyeLandmarks = 142;
|
||||
|
||||
constexpr int kEyeLandmarkIndicesInFaceLandmarks[] = {
|
||||
// Left eye
|
||||
// eye lower contour
|
||||
33,
|
||||
7,
|
||||
163,
|
||||
144,
|
||||
145,
|
||||
153,
|
||||
154,
|
||||
155,
|
||||
133,
|
||||
// eye upper contour (excluding corners)
|
||||
246,
|
||||
161,
|
||||
160,
|
||||
159,
|
||||
158,
|
||||
157,
|
||||
173,
|
||||
// halo x2 lower contour
|
||||
130,
|
||||
25,
|
||||
110,
|
||||
24,
|
||||
23,
|
||||
22,
|
||||
26,
|
||||
112,
|
||||
243,
|
||||
// halo x2 upper contour (excluding corners)
|
||||
247,
|
||||
30,
|
||||
29,
|
||||
27,
|
||||
28,
|
||||
56,
|
||||
190,
|
||||
// halo x3 lower contour
|
||||
226,
|
||||
31,
|
||||
228,
|
||||
229,
|
||||
230,
|
||||
231,
|
||||
232,
|
||||
233,
|
||||
244,
|
||||
// halo x3 upper contour (excluding corners)
|
||||
113,
|
||||
225,
|
||||
224,
|
||||
223,
|
||||
222,
|
||||
221,
|
||||
189,
|
||||
// halo x4 upper contour (no lower because of mesh structure)
|
||||
// or eyebrow inner contour
|
||||
35,
|
||||
124,
|
||||
46,
|
||||
53,
|
||||
52,
|
||||
65,
|
||||
// halo x5 lower contour
|
||||
143,
|
||||
111,
|
||||
117,
|
||||
118,
|
||||
119,
|
||||
120,
|
||||
121,
|
||||
128,
|
||||
245,
|
||||
// halo x5 upper contour (excluding corners)
|
||||
// or eyebrow outer contour
|
||||
156,
|
||||
70,
|
||||
63,
|
||||
105,
|
||||
66,
|
||||
107,
|
||||
55,
|
||||
193,
|
||||
|
||||
// Right eye
|
||||
// eye lower contour
|
||||
263,
|
||||
249,
|
||||
390,
|
||||
373,
|
||||
374,
|
||||
380,
|
||||
381,
|
||||
382,
|
||||
362,
|
||||
// eye upper contour (excluding corners)
|
||||
466,
|
||||
388,
|
||||
387,
|
||||
386,
|
||||
385,
|
||||
384,
|
||||
398,
|
||||
// halo x2 lower contour
|
||||
359,
|
||||
255,
|
||||
339,
|
||||
254,
|
||||
253,
|
||||
252,
|
||||
256,
|
||||
341,
|
||||
463,
|
||||
// halo x2 upper contour (excluding corners)
|
||||
467,
|
||||
260,
|
||||
259,
|
||||
257,
|
||||
258,
|
||||
286,
|
||||
414,
|
||||
// halo x3 lower contour
|
||||
446,
|
||||
261,
|
||||
448,
|
||||
449,
|
||||
450,
|
||||
451,
|
||||
452,
|
||||
453,
|
||||
464,
|
||||
// halo x3 upper contour (excluding corners)
|
||||
342,
|
||||
445,
|
||||
444,
|
||||
443,
|
||||
442,
|
||||
441,
|
||||
413,
|
||||
// halo x4 upper contour (no lower because of mesh structure)
|
||||
// or eyebrow inner contour
|
||||
265,
|
||||
353,
|
||||
276,
|
||||
283,
|
||||
282,
|
||||
295,
|
||||
// halo x5 lower contour
|
||||
372,
|
||||
340,
|
||||
346,
|
||||
347,
|
||||
348,
|
||||
349,
|
||||
350,
|
||||
357,
|
||||
465,
|
||||
// halo x5 upper contour (excluding corners)
|
||||
// or eyebrow outer contour
|
||||
383,
|
||||
300,
|
||||
293,
|
||||
334,
|
||||
296,
|
||||
336,
|
||||
285,
|
||||
417,
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
// Update face landmarks with new (e.g., refined) values. Currently only updates
|
||||
// landmarks around the eyes.
|
||||
//
|
||||
// Usage example:
|
||||
// node {
|
||||
// calculator: "UpdateFaceLandmarksCalculator"
|
||||
// input_stream: "NEW_EYE_LANDMARKS:new_eye_landmarks"
|
||||
// input_stream: "FACE_LANDMARKS:face_landmarks"
|
||||
// output_stream: "UPDATED_FACE_LANDMARKS:refine_face_landmarks"
|
||||
// }
|
||||
//
|
||||
class UpdateFaceLandmarksCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kFaceLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kNewEyeLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
|
||||
cc->Outputs().Tag(kUpdatedFaceLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
::mediapipe::Status Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(UpdateFaceLandmarksCalculator);
|
||||
|
||||
::mediapipe::Status UpdateFaceLandmarksCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
if (cc->Inputs().Tag(kFaceLandmarksTag).IsEmpty() ||
|
||||
cc->Inputs().Tag(kNewEyeLandmarksTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
const auto& face_landmarks =
|
||||
cc->Inputs().Tag(kFaceLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
const auto& new_eye_landmarks =
|
||||
cc->Inputs().Tag(kNewEyeLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
|
||||
RET_CHECK_EQ(face_landmarks.landmark_size(), kNumFaceLandmarks)
|
||||
<< "Wrong number of face landmarks";
|
||||
RET_CHECK_EQ(new_eye_landmarks.landmark_size(), kNumEyeLandmarks)
|
||||
<< "Wrong number of face landmarks";
|
||||
|
||||
auto refined_face_landmarks =
|
||||
absl::make_unique<NormalizedLandmarkList>(face_landmarks);
|
||||
for (int i = 0; i < kNumEyeLandmarks; ++i) {
|
||||
const auto& refined_ld = new_eye_landmarks.landmark(i);
|
||||
const int id = kEyeLandmarkIndicesInFaceLandmarks[i];
|
||||
refined_face_landmarks->mutable_landmark(id)->set_x(refined_ld.x());
|
||||
refined_face_landmarks->mutable_landmark(id)->set_y(refined_ld.y());
|
||||
refined_face_landmarks->mutable_landmark(id)->set_z(refined_ld.z());
|
||||
refined_face_landmarks->mutable_landmark(id)->set_visibility(
|
||||
refined_ld.visibility());
|
||||
}
|
||||
cc->Outputs()
|
||||
.Tag(kUpdatedFaceLandmarksTag)
|
||||
.Add(refined_face_landmarks.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -124,11 +124,25 @@ node {
|
||||
output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ConcatenateNormalizedLandmarkListCalculator"
|
||||
input_stream: "left_eye_contour_landmarks"
|
||||
input_stream: "right_eye_contour_landmarks"
|
||||
output_stream: "refined_eye_landmarks"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "UpdateFaceLandmarksCalculator"
|
||||
input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks"
|
||||
input_stream: "FACE_LANDMARKS:face_landmarks"
|
||||
output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks"
|
||||
}
|
||||
|
||||
# Renders annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "IrisAndDepthRendererCpu"
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_stream: "FACE_LANDMARKS:face_landmarks"
|
||||
input_stream: "FACE_LANDMARKS:updated_face_landmarks"
|
||||
input_stream: "EYE_LANDMARKS_LEFT:left_eye_contour_landmarks"
|
||||
input_stream: "EYE_LANDMARKS_RIGHT:right_eye_contour_landmarks"
|
||||
input_stream: "IRIS_LANDMARKS_LEFT:left_iris_landmarks"
|
||||
@@ -138,6 +152,7 @@ node {
|
||||
input_stream: "RIGHT_EYE_RECT:right_eye_rect_from_landmarks"
|
||||
input_stream: "DETECTIONS:face_detections"
|
||||
input_side_packet: "IMAGE_FILE_PROPERTIES:image_file_properties"
|
||||
output_stream: "IRIS_LANDMARKS:iris_landmarks"
|
||||
output_stream: "IMAGE:output_image"
|
||||
output_stream: "LEFT_IRIS_DEPTH_MM:left_iris_depth_mm"
|
||||
output_stream: "RIGHT_IRIS_DEPTH_MM:right_iris_depth_mm"
|
||||
|
||||
@@ -8,6 +8,8 @@ input_stream: "input_video"
|
||||
|
||||
# CPU image. (ImageFrame)
|
||||
output_stream: "output_video"
|
||||
# Face landmarks with iris. (NormalizedLandmarkList)
|
||||
output_stream: "face_landmarks_with_iris"
|
||||
|
||||
# Defines how many faces to detect. Iris tracking currently only handles one
|
||||
# face (left and right eye), and therefore this should always be set to 1.
|
||||
@@ -101,11 +103,25 @@ node {
|
||||
output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ConcatenateNormalizedLandmarkListCalculator"
|
||||
input_stream: "left_eye_contour_landmarks"
|
||||
input_stream: "right_eye_contour_landmarks"
|
||||
output_stream: "refined_eye_landmarks"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "UpdateFaceLandmarksCalculator"
|
||||
input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks"
|
||||
input_stream: "FACE_LANDMARKS:face_landmarks"
|
||||
output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks"
|
||||
}
|
||||
|
||||
# Renders annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "IrisRendererCpu"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "FACE_LANDMARKS:face_landmarks"
|
||||
input_stream: "FACE_LANDMARKS:updated_face_landmarks"
|
||||
input_stream: "EYE_LANDMARKS_LEFT:left_eye_contour_landmarks"
|
||||
input_stream: "EYE_LANDMARKS_RIGHT:right_eye_contour_landmarks"
|
||||
input_stream: "IRIS_LANDMARKS_LEFT:left_iris_landmarks"
|
||||
@@ -114,5 +130,13 @@ node {
|
||||
input_stream: "LEFT_EYE_RECT:left_eye_rect_from_landmarks"
|
||||
input_stream: "RIGHT_EYE_RECT:right_eye_rect_from_landmarks"
|
||||
input_stream: "DETECTIONS:face_detections"
|
||||
output_stream: "IRIS_LANDMARKS:iris_landmarks"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ConcatenateNormalizedLandmarkListCalculator"
|
||||
input_stream: "updated_face_landmarks"
|
||||
input_stream: "iris_landmarks"
|
||||
output_stream: "face_landmarks_with_iris"
|
||||
}
|
||||
|
||||
@@ -106,11 +106,25 @@ node {
|
||||
output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ConcatenateNormalizedLandmarkListCalculator"
|
||||
input_stream: "left_eye_contour_landmarks"
|
||||
input_stream: "right_eye_contour_landmarks"
|
||||
output_stream: "refined_eye_landmarks"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "UpdateFaceLandmarksCalculator"
|
||||
input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks"
|
||||
input_stream: "FACE_LANDMARKS:face_landmarks"
|
||||
output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks"
|
||||
}
|
||||
|
||||
# Renders annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "IrisRendererCpu"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "FACE_LANDMARKS:face_landmarks"
|
||||
input_stream: "FACE_LANDMARKS:updated_face_landmarks"
|
||||
input_stream: "EYE_LANDMARKS_LEFT:left_eye_contour_landmarks"
|
||||
input_stream: "EYE_LANDMARKS_RIGHT:right_eye_contour_landmarks"
|
||||
input_stream: "IRIS_LANDMARKS_LEFT:left_iris_landmarks"
|
||||
@@ -119,6 +133,7 @@ node {
|
||||
input_stream: "LEFT_EYE_RECT:left_eye_rect_from_landmarks"
|
||||
input_stream: "RIGHT_EYE_RECT:right_eye_rect_from_landmarks"
|
||||
input_stream: "DETECTIONS:face_detections"
|
||||
output_stream: "IRIS_LANDMARKS:iris_landmarks"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ input_stream: "input_video"
|
||||
|
||||
# GPU buffer. (GpuBuffer)
|
||||
output_stream: "output_video"
|
||||
# Face landmarks with iris. (NormalizedLandmarkList)
|
||||
output_stream: "face_landmarks_with_iris"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for downstream nodes
|
||||
@@ -121,11 +123,25 @@ node {
|
||||
output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ConcatenateNormalizedLandmarkListCalculator"
|
||||
input_stream: "left_eye_contour_landmarks"
|
||||
input_stream: "right_eye_contour_landmarks"
|
||||
output_stream: "refined_eye_landmarks"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "UpdateFaceLandmarksCalculator"
|
||||
input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks"
|
||||
input_stream: "FACE_LANDMARKS:face_landmarks"
|
||||
output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks"
|
||||
}
|
||||
|
||||
# Renders annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "IrisAndDepthRendererGpu"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "FACE_LANDMARKS:face_landmarks"
|
||||
input_stream: "FACE_LANDMARKS:updated_face_landmarks"
|
||||
input_stream: "EYE_LANDMARKS_LEFT:left_eye_contour_landmarks"
|
||||
input_stream: "EYE_LANDMARKS_RIGHT:right_eye_contour_landmarks"
|
||||
input_stream: "IRIS_LANDMARKS_LEFT:left_iris_landmarks"
|
||||
@@ -138,3 +154,10 @@ node {
|
||||
output_stream: "IRIS_LANDMARKS:iris_landmarks"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ConcatenateNormalizedLandmarkListCalculator"
|
||||
input_stream: "updated_face_landmarks"
|
||||
input_stream: "iris_landmarks"
|
||||
output_stream: "face_landmarks_with_iris"
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
@@ -32,6 +32,8 @@ cc_library(
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:rect_to_render_data_calculator",
|
||||
"//mediapipe/graphs/face_mesh/calculators:face_landmarks_to_render_data_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:iris_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -42,7 +44,6 @@ mediapipe_simple_subgraph(
|
||||
deps = [
|
||||
":renderer_calculators",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:iris_to_depth_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:iris_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -52,7 +53,6 @@ mediapipe_simple_subgraph(
|
||||
register_as = "IrisRendererCpu",
|
||||
deps = [
|
||||
":renderer_calculators",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:iris_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -63,6 +63,5 @@ mediapipe_simple_subgraph(
|
||||
deps = [
|
||||
":renderer_calculators",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:iris_to_depth_calculator",
|
||||
"//mediapipe/graphs/iris_tracking/calculators:iris_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -13,6 +13,7 @@ input_stream: "NORM_RECT:rect"
|
||||
input_stream: "LEFT_EYE_RECT:left_eye_rect_from_landmarks"
|
||||
input_stream: "RIGHT_EYE_RECT:right_eye_rect_from_landmarks"
|
||||
input_side_packet: "IMAGE_FILE_PROPERTIES:image_file_properties"
|
||||
output_stream: "IRIS_LANDMARKS:iris_landmarks"
|
||||
output_stream: "IMAGE:output_image"
|
||||
output_stream: "LEFT_IRIS_DEPTH_MM:left_iris_depth_mm"
|
||||
output_stream: "RIGHT_IRIS_DEPTH_MM:right_iris_depth_mm"
|
||||
@@ -47,13 +48,19 @@ node {
|
||||
output_stream: "iris_landmarks"
|
||||
}
|
||||
|
||||
# Concatenate iris landmarks from both eyes and face landmarks.
|
||||
# Converts landmarks to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "ConcatenateNormalizedLandmarkListCalculator"
|
||||
input_stream: "left_iris_landmarks"
|
||||
input_stream: "right_iris_landmarks"
|
||||
input_stream: "face_landmarks"
|
||||
output_stream: "face_iris_landmarks"
|
||||
calculator: "FaceLandmarksToRenderDataCalculator"
|
||||
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||
output_stream: "RENDER_DATA:face_landmarks_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||
landmark_color { r: 150 g: 0 b: 0 }
|
||||
connection_color { r: 0 g: 150 b: 0 }
|
||||
thickness: 2
|
||||
visualize_landmark_depth: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
@@ -252,6 +259,7 @@ node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "face_landmarks_render_data"
|
||||
input_stream: "right_eye_contour_landmarks_render_data"
|
||||
input_stream: "left_eye_contour_landmarks_render_data"
|
||||
input_stream: "iris_render_data"
|
||||
|
||||
@@ -46,13 +46,19 @@ node {
|
||||
output_stream: "iris_landmarks"
|
||||
}
|
||||
|
||||
# Concatenate iris landmarks from both eyes and face landmarks.
|
||||
# Converts landmarks to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "ConcatenateNormalizedLandmarkListCalculator"
|
||||
input_stream: "left_iris_landmarks"
|
||||
input_stream: "right_iris_landmarks"
|
||||
input_stream: "face_landmarks"
|
||||
output_stream: "face_iris_landmarks"
|
||||
calculator: "FaceLandmarksToRenderDataCalculator"
|
||||
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||
output_stream: "RENDER_DATA:face_landmarks_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||
landmark_color { r: 150 g: 0 b: 0 }
|
||||
connection_color { r: 0 g: 150 b: 0 }
|
||||
thickness: 2
|
||||
visualize_landmark_depth: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
@@ -251,6 +257,7 @@ node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "IMAGE_GPU:input_image"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "face_landmarks_render_data"
|
||||
input_stream: "right_eye_contour_landmarks_render_data"
|
||||
input_stream: "left_eye_contour_landmarks_render_data"
|
||||
input_stream: "iris_render_data"
|
||||
|
||||
@@ -12,6 +12,7 @@ input_stream: "IRIS_LANDMARKS_RIGHT:right_iris_landmarks"
|
||||
input_stream: "NORM_RECT:rect"
|
||||
input_stream: "LEFT_EYE_RECT:left_eye_rect_from_landmarks"
|
||||
input_stream: "RIGHT_EYE_RECT:right_eye_rect_from_landmarks"
|
||||
output_stream: "IRIS_LANDMARKS:iris_landmarks"
|
||||
output_stream: "IMAGE:output_image"
|
||||
|
||||
node {
|
||||
@@ -44,15 +45,22 @@ node {
|
||||
output_stream: "iris_landmarks"
|
||||
}
|
||||
|
||||
# Concatenate iris landmarks from both eyes and face landmarks.
|
||||
# Converts landmarks to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "ConcatenateNormalizedLandmarkListCalculator"
|
||||
input_stream: "left_iris_landmarks"
|
||||
input_stream: "right_iris_landmarks"
|
||||
input_stream: "face_landmarks"
|
||||
output_stream: "face_iris_landmarks"
|
||||
calculator: "FaceLandmarksToRenderDataCalculator"
|
||||
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||
output_stream: "RENDER_DATA:face_landmarks_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||
landmark_color { r: 150 g: 0 b: 0 }
|
||||
connection_color { r: 0 g: 150 b: 0 }
|
||||
thickness: 2
|
||||
visualize_landmark_depth: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE:input_image"
|
||||
@@ -238,6 +246,7 @@ node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "face_landmarks_render_data"
|
||||
input_stream: "right_eye_contour_landmarks_render_data"
|
||||
input_stream: "left_eye_contour_landmarks_render_data"
|
||||
input_stream: "iris_render_data"
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
@@ -319,6 +319,7 @@ cc_library(
|
||||
"model.h",
|
||||
"types.h",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":annotation_cc_proto",
|
||||
":object_cc_proto",
|
||||
|
||||
+193
-16
@@ -1,4 +1,4 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
// Copyright 2020 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -47,7 +47,7 @@ typedef ImageFrame AssetTextureFormat;
|
||||
typedef GpuBuffer AssetTextureFormat;
|
||||
#endif
|
||||
|
||||
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
|
||||
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, ATTRIB_NORMAL, NUM_ATTRIBUTES };
|
||||
static const int kNumMatrixEntries = 16;
|
||||
|
||||
// Hard-coded MVP Matrix for testing.
|
||||
@@ -68,10 +68,14 @@ static const float kModelMatrix[] = {0.83704215, -0.36174262, 0.41049102, 0.0,
|
||||
// MODEL_MATRICES (TimedModelMatrixProtoList, optional):
|
||||
// If provided, will set the model matrices for the objects to be rendered
|
||||
// during future rendering calls.
|
||||
// TEXTURE (ImageFrame on Android / GpuBuffer on iOS, semi-optional):
|
||||
// Texture to use with animation file. Texture is REQUIRED to be passed into
|
||||
// the calculator, but can be passed in as a Side Packet OR Input Stream.
|
||||
//
|
||||
// Input side packets:
|
||||
// TEXTURE (ImageFrame on Android / GpuBuffer on iOS, required):
|
||||
// Texture to use with animation file.
|
||||
// TEXTURE (ImageFrame on Android / GpuBuffer on iOS, semi-optional):
|
||||
// Texture to use with animation file. Texture is REQUIRED to be passed into
|
||||
// the calculator, but can be passed in as a Side Packet OR Input Stream.
|
||||
// ANIMATION_ASSET (String, required):
|
||||
// Path of animation file to load and render. Should be generated by
|
||||
// //java/com/google/android/apps/motionstills/SimpleObjEncryptor with
|
||||
@@ -98,9 +102,9 @@ static const float kModelMatrix[] = {0.83704215, -0.36174262, 0.41049102, 0.0,
|
||||
|
||||
// Simple helper-struct for containing the parsed geometry data from a 3D
|
||||
// animation frame for rendering.
|
||||
|
||||
struct TriangleMesh {
|
||||
int index_count = 0; // Needed for glDrawElements rendering call
|
||||
std::unique_ptr<float[]> normals = nullptr;
|
||||
std::unique_ptr<float[]> vertices = nullptr;
|
||||
std::unique_ptr<float[]> texture_coords = nullptr;
|
||||
std::unique_ptr<int16[]> triangle_indices = nullptr;
|
||||
@@ -156,6 +160,7 @@ class GlAnimationOverlayCalculator : public CalculatorBase {
|
||||
void ComputeAspectRatioAndFovFromCameraParameters(
|
||||
const CameraParametersProto &camera_parameters, float *aspect_ratio,
|
||||
float *vertical_fov_degrees);
|
||||
|
||||
int GetAnimationFrameIndex(Timestamp timestamp);
|
||||
::mediapipe::Status GlSetup();
|
||||
::mediapipe::Status GlBind(const TriangleMesh &triangle_mesh,
|
||||
@@ -167,6 +172,9 @@ class GlAnimationOverlayCalculator : public CalculatorBase {
|
||||
float z_far);
|
||||
void LoadModelMatrices(const TimedModelMatrixProtoList &model_matrices,
|
||||
std::vector<ModelMatrix> *current_model_matrices);
|
||||
void CalculateTriangleMeshNormals(int normals_len,
|
||||
TriangleMesh *triangle_mesh);
|
||||
void Normalize3f(float input[3]);
|
||||
|
||||
#if !defined(__ANDROID__)
|
||||
// Asset loading routine for all non-Android platforms.
|
||||
@@ -198,7 +206,13 @@ REGISTER_CALCULATOR(GlAnimationOverlayCalculator);
|
||||
cc->Inputs().Tag("MASK_MODEL_MATRICES").Set<TimedModelMatrixProtoList>();
|
||||
}
|
||||
|
||||
cc->InputSidePackets().Tag("TEXTURE").Set<AssetTextureFormat>();
|
||||
// Must have texture as Input Stream or Side Packet
|
||||
if (cc->InputSidePackets().HasTag("TEXTURE")) {
|
||||
cc->InputSidePackets().Tag("TEXTURE").Set<AssetTextureFormat>();
|
||||
} else {
|
||||
cc->Inputs().Tag("TEXTURE").Set<AssetTextureFormat>();
|
||||
}
|
||||
|
||||
cc->InputSidePackets().Tag("ANIMATION_ASSET").Set<std::string>();
|
||||
if (cc->InputSidePackets().HasTag("CAMERA_PARAMETERS_PROTO_STRING")) {
|
||||
cc->InputSidePackets()
|
||||
@@ -216,6 +230,83 @@ REGISTER_CALCULATOR(GlAnimationOverlayCalculator);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
void GlAnimationOverlayCalculator::CalculateTriangleMeshNormals(
|
||||
int normals_len, TriangleMesh *triangle_mesh) {
|
||||
// Set triangle_mesh normals for shader usage
|
||||
triangle_mesh->normals.reset(new float[normals_len]);
|
||||
// Used for storing the vertex normals prior to averaging
|
||||
std::vector<float> vertex_normals_sum(normals_len, 0.0f);
|
||||
// Compute every triangle surface normal and store them for averaging
|
||||
for (int idx = 0; idx < triangle_mesh->index_count; idx += 3) {
|
||||
int v_idx[3];
|
||||
v_idx[0] = triangle_mesh->triangle_indices.get()[idx];
|
||||
v_idx[1] = triangle_mesh->triangle_indices.get()[idx + 1];
|
||||
v_idx[2] = triangle_mesh->triangle_indices.get()[idx + 2];
|
||||
// (V1) vertex X,Y,Z indices in triangle_mesh.vertices
|
||||
const float v1x = triangle_mesh->vertices[v_idx[0] * 3];
|
||||
const float v1y = triangle_mesh->vertices[v_idx[0] * 3 + 1];
|
||||
const float v1z = triangle_mesh->vertices[v_idx[0] * 3 + 2];
|
||||
// (V2) vertex X,Y,Z indices in triangle_mesh.vertices
|
||||
const float v2x = triangle_mesh->vertices[v_idx[1] * 3];
|
||||
const float v2y = triangle_mesh->vertices[v_idx[1] * 3 + 1];
|
||||
const float v2z = triangle_mesh->vertices[v_idx[1] * 3 + 2];
|
||||
// (V3) vertex X,Y,Z indices in triangle_mesh.vertices
|
||||
const float v3x = triangle_mesh->vertices[v_idx[2] * 3];
|
||||
const float v3y = triangle_mesh->vertices[v_idx[2] * 3 + 1];
|
||||
const float v3z = triangle_mesh->vertices[v_idx[2] * 3 + 2];
|
||||
// Calculate normals from vertices
|
||||
// V2 - V1
|
||||
const float ax = v2x - v1x;
|
||||
const float ay = v2y - v1y;
|
||||
const float az = v2z - v1z;
|
||||
// V3 - V1
|
||||
const float bx = v3x - v1x;
|
||||
const float by = v3y - v1y;
|
||||
const float bz = v3z - v1z;
|
||||
// Calculate cross product
|
||||
const float normal_x = ay * bz - az * by;
|
||||
const float normal_y = az * bx - ax * bz;
|
||||
const float normal_z = ax * by - ay * bx;
|
||||
// The normals calculated above must be normalized if we wish to prevent
|
||||
// triangles with a larger surface area from dominating the normal
|
||||
// calculations, however, none of our current models require this
|
||||
// normalization.
|
||||
|
||||
// Add connected normal to each associated vertex
|
||||
// It is also necessary to increment each vertex denominator for averaging
|
||||
for (int i = 0; i < 3; i++) {
|
||||
vertex_normals_sum[v_idx[i] * 3] += normal_x;
|
||||
vertex_normals_sum[v_idx[i] * 3 + 1] += normal_y;
|
||||
vertex_normals_sum[v_idx[i] * 3 + 2] += normal_z;
|
||||
}
|
||||
}
|
||||
|
||||
// Combine all triangle normals connected to each vertex by adding the X,Y,Z
|
||||
// value of each adjacent triangle surface normal to every vertex and then
|
||||
// averaging the combined value.
|
||||
for (int idx = 0; idx < normals_len; idx += 3) {
|
||||
float normal[3];
|
||||
normal[0] = vertex_normals_sum[idx];
|
||||
normal[1] = vertex_normals_sum[idx + 1];
|
||||
normal[2] = vertex_normals_sum[idx + 2];
|
||||
Normalize3f(normal);
|
||||
triangle_mesh->normals.get()[idx] = normal[0];
|
||||
triangle_mesh->normals.get()[idx + 1] = normal[1];
|
||||
triangle_mesh->normals.get()[idx + 2] = normal[2];
|
||||
}
|
||||
}
|
||||
|
||||
void GlAnimationOverlayCalculator::Normalize3f(float input[3]) {
|
||||
float product = 0.0;
|
||||
product += input[0] * input[0];
|
||||
product += input[1] * input[1];
|
||||
product += input[2] * input[2];
|
||||
float magnitude = sqrt(product);
|
||||
input[0] /= magnitude;
|
||||
input[1] /= magnitude;
|
||||
input[2] /= magnitude;
|
||||
}
|
||||
|
||||
// Helper function for initializing our perspective matrix.
|
||||
void GlAnimationOverlayCalculator::InitializePerspectiveMatrix(
|
||||
float aspect_ratio, float fov_degrees, float z_near, float z_far) {
|
||||
@@ -319,6 +410,10 @@ bool GlAnimationOverlayCalculator::LoadAnimationAndroid(
|
||||
LOG(ERROR) << "Failed to read indices for frame " << frame_count_;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Set the normals for this triangle_mesh
|
||||
CalculateTriangleMeshNormals(lengths[0], &triangle_mesh);
|
||||
|
||||
frame_count_++;
|
||||
}
|
||||
AAsset_close(asset);
|
||||
@@ -383,6 +478,10 @@ bool GlAnimationOverlayCalculator::LoadAnimation(const std::string &filename) {
|
||||
<< frame_count_;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Set the normals for this triangle_mesh
|
||||
CalculateTriangleMeshNormals(lengths[0], &triangle_mesh);
|
||||
|
||||
frame_count_++;
|
||||
}
|
||||
|
||||
@@ -473,10 +572,13 @@ void GlAnimationOverlayCalculator::ComputeAspectRatioAndFovFromCameraParameters(
|
||||
mask_texture_ = helper_.CreateSourceTexture(mask_texture);
|
||||
}
|
||||
|
||||
// Load in our asset's texture data
|
||||
const auto &input_texture =
|
||||
cc->InputSidePackets().Tag("TEXTURE").Get<AssetTextureFormat>();
|
||||
texture_ = helper_.CreateSourceTexture(input_texture);
|
||||
// Load in all static texture data if it exists
|
||||
if (cc->InputSidePackets().HasTag("TEXTURE")) {
|
||||
const auto &input_texture =
|
||||
cc->InputSidePackets().Tag("TEXTURE").Get<AssetTextureFormat>();
|
||||
texture_ = helper_.CreateSourceTexture(input_texture);
|
||||
}
|
||||
|
||||
VLOG(2) << "Input texture size: " << texture_.width() << ", "
|
||||
<< texture_.height() << std::endl;
|
||||
|
||||
@@ -604,6 +706,14 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
|
||||
int frame_index = GetAnimationFrameIndex(cc->InputTimestamp());
|
||||
const TriangleMesh ¤t_frame = triangle_meshes_[frame_index];
|
||||
|
||||
// Load dynamic texture if it exists
|
||||
if (cc->Inputs().HasTag("TEXTURE")) {
|
||||
const auto &input_texture =
|
||||
cc->Inputs().Tag("TEXTURE").Get<AssetTextureFormat>();
|
||||
texture_ = helper_.CreateSourceTexture(input_texture);
|
||||
}
|
||||
|
||||
MP_RETURN_IF_ERROR(GlBind(current_frame, texture_));
|
||||
if (has_model_matrix_stream_) {
|
||||
// Draw objects using our latest model matrix stream packet.
|
||||
@@ -616,8 +726,9 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
}
|
||||
|
||||
// Disable vertex attributes
|
||||
GLCHECK(glEnableVertexAttribArray(ATTRIB_VERTEX));
|
||||
GLCHECK(glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION));
|
||||
GLCHECK(glDisableVertexAttribArray(ATTRIB_VERTEX));
|
||||
GLCHECK(glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION));
|
||||
GLCHECK(glDisableVertexAttribArray(ATTRIB_NORMAL));
|
||||
|
||||
// Disable depth test
|
||||
GLCHECK(glDisable(GL_DEPTH_TEST));
|
||||
@@ -645,10 +756,12 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
const GLint attr_location[NUM_ATTRIBUTES] = {
|
||||
ATTRIB_VERTEX,
|
||||
ATTRIB_TEXTURE_POSITION,
|
||||
ATTRIB_NORMAL,
|
||||
};
|
||||
const GLchar *attr_name[NUM_ATTRIBUTES] = {
|
||||
"position",
|
||||
"texture_coordinate",
|
||||
"normal",
|
||||
};
|
||||
|
||||
const GLchar *vert_src = R"(
|
||||
@@ -660,28 +773,89 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
|
||||
// vertex position in threespace
|
||||
attribute vec4 position;
|
||||
attribute vec3 normal;
|
||||
|
||||
// texture coordinate for each vertex in normalized texture space (0..1)
|
||||
attribute mediump vec4 texture_coordinate;
|
||||
|
||||
// texture coordinate for fragment shader (will be interpolated)
|
||||
varying mediump vec2 sample_coordinate;
|
||||
varying mediump vec2 sampleCoordinate;
|
||||
varying mediump vec3 vNormal;
|
||||
|
||||
void main() {
|
||||
sample_coordinate = texture_coordinate.xy;
|
||||
sampleCoordinate = texture_coordinate.xy;
|
||||
mat4 mvpMatrix = perspectiveMatrix * modelMatrix;
|
||||
gl_Position = mvpMatrix * position;
|
||||
|
||||
// TODO: Pass in rotation submatrix with no scaling or transforms to prevent
|
||||
// breaking vNormal in case of model matrix having non-uniform scaling
|
||||
vec4 tmpNormal = mvpMatrix * vec4(normal, 1.0);
|
||||
vec4 transformedZero = mvpMatrix * vec4(0.0, 0.0, 0.0, 1.0);
|
||||
tmpNormal = tmpNormal - transformedZero;
|
||||
vNormal = normalize(tmpNormal.xyz);
|
||||
}
|
||||
)";
|
||||
|
||||
const GLchar *frag_src = R"(
|
||||
precision mediump float;
|
||||
|
||||
varying vec2 sample_coordinate; // texture coordinate (0..1)
|
||||
varying vec2 sampleCoordinate; // texture coordinate (0..1)
|
||||
varying vec3 vNormal;
|
||||
uniform sampler2D texture; // texture to shade with
|
||||
const float kPi = 3.14159265359;
|
||||
|
||||
// Define ambient lighting factor that is applied to our texture in order to
|
||||
// generate ambient lighting of the scene on the object. Range is [0.0-1.0],
|
||||
// with the factor being proportional to the brightness of the lighting in the
|
||||
// scene being applied to the object
|
||||
const float kAmbientLighting = 0.75;
|
||||
|
||||
// Define RGB values for light source
|
||||
const vec3 kLightColor = vec3(0.25);
|
||||
// Exponent for directional lighting that governs diffusion of surface light
|
||||
const float kExponent = 1.0;
|
||||
// Define direction of lighting effect source
|
||||
const vec3 lightDir = vec3(0.0, -1.0, -0.6);
|
||||
// Hard-coded view direction
|
||||
const vec3 viewDir = vec3(0.0, 0.0, -1.0);
|
||||
|
||||
// DirectionalLighting procedure imported from Lullaby @ https://github.com/google/lullaby
|
||||
// Calculate and return the color (diffuse and specular together) reflected by
|
||||
// a directional light.
|
||||
vec3 GetDirectionalLight(vec3 pos, vec3 normal, vec3 viewDir, vec3 lightDir, vec3 lightColor, float exponent) {
|
||||
// Intensity of the diffuse light. Saturate to keep within the 0-1 range.
|
||||
float normal_dot_light_dir = dot(-normal, -lightDir);
|
||||
float intensity = clamp(normal_dot_light_dir, 0.0, 1.0);
|
||||
// Calculate the diffuse light
|
||||
vec3 diffuse = intensity * lightColor;
|
||||
// http://www.rorydriscoll.com/2009/01/25/energy-conservation-in-games/
|
||||
float kEnergyConservation = (2.0 + exponent) / (2.0 * kPi);
|
||||
vec3 reflect_dir = reflect(lightDir, -normal);
|
||||
// Intensity of the specular light
|
||||
float view_dot_reflect = dot(-viewDir, reflect_dir);
|
||||
// Use an epsilon for pow because pow(x,y) is undefined if x < 0 or x == 0
|
||||
// and y <= 0 (GLSL Spec 8.2)
|
||||
const float kEpsilon = 1e-5;
|
||||
intensity = kEnergyConservation * pow(clamp(view_dot_reflect, kEpsilon, 1.0),
|
||||
exponent);
|
||||
// Specular color:
|
||||
vec3 specular = intensity * lightColor;
|
||||
return diffuse + specular;
|
||||
}
|
||||
|
||||
void main() {
|
||||
gl_FragColor = texture2D(texture, sample_coordinate);
|
||||
// Sample the texture, retrieving an rgba pixel value
|
||||
vec4 pixel = texture2D(texture, sampleCoordinate);
|
||||
// If the alpha (background) value is near transparent, then discard the
|
||||
// pixel, this allows the rendering of transparent background GIFs
|
||||
// TODO: Adding a toggle to perform pixel alpha discarding for transparent
|
||||
// GIFs (prevent interference with Objectron system).
|
||||
if (pixel.a < 0.2) discard;
|
||||
|
||||
// Generate directional lighting effect
|
||||
vec3 lighting = GetDirectionalLight(gl_FragCoord.xyz, vNormal, viewDir, lightDir, kLightColor, kExponent);
|
||||
// Apply both ambient and directional lighting to our texture
|
||||
gl_FragColor = vec4((vec3(kAmbientLighting) + lighting) * pixel.rgb, 1.0);
|
||||
}
|
||||
)";
|
||||
|
||||
@@ -718,6 +892,9 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
|
||||
GLCHECK(glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0,
|
||||
triangle_mesh.texture_coords.get()));
|
||||
GLCHECK(glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION));
|
||||
GLCHECK(glVertexAttribPointer(ATTRIB_NORMAL, 3, GL_FLOAT, 0, 0,
|
||||
triangle_mesh.normals.get()));
|
||||
GLCHECK(glEnableVertexAttribArray(ATTRIB_NORMAL));
|
||||
GLCHECK(glActiveTexture(GL_TEXTURE1));
|
||||
GLCHECK(glBindTexture(texture.target(), texture.name()));
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ load(
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user