talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: ea8d45731f5a052f79745e35bfd8240d6ac568d2
This commit is contained in:
MediaPipe Team
2020-12-16 00:05:25 -05:00
committed by chuoling
parent 38be2ec58f
commit 39309bedba
109 changed files with 5803 additions and 1500 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/modules/holistic_landmark/hand_recrop_by_roi_cpu.pbtxt
+1
View File
@@ -43,6 +43,7 @@ node {
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/holistic_landmark/hand_recrop.tflite"
delegate { xnnpack {} }
}
}
}
mediapipe/modules/objectron/BUILD
+33 -29
View File
@@ -56,25 +56,19 @@ mediapipe_simple_subgraph(
graph = "box_landmark_gpu.pbtxt",
register_as = "BoxLandmarkSubgraph",
deps = [
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/image:image_cropping_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/tflite:tflite_converter_calculator",
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
"//mediapipe/calculators/tflite:tflite_inference_calculator",
"//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
"//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
"//mediapipe/modules/objectron/calculators:frame_annotation_to_rect_calculator",
"//mediapipe/modules/objectron/calculators:landmarks_to_frame_annotation_calculator",
"//mediapipe/modules/objectron/calculators:lift_2d_frame_annotation_to_3d_calculator",
],
)
@@ -83,24 +77,19 @@ mediapipe_simple_subgraph(
graph = "box_landmark_cpu.pbtxt",
register_as = "BoxLandmarkSubgraph",
deps = [
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/image:image_cropping_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/tflite:tflite_converter_calculator",
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
"//mediapipe/calculators/tflite:tflite_inference_calculator",
"//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
"//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
"//mediapipe/modules/objectron/calculators:frame_annotation_to_rect_calculator",
"//mediapipe/modules/objectron/calculators:landmarks_to_frame_annotation_calculator",
"//mediapipe/modules/objectron/calculators:lift_2d_frame_annotation_to_3d_calculator",
],
)
@@ -115,9 +104,7 @@ mediapipe_simple_subgraph(
"//mediapipe/calculators/tflite:tflite_inference_calculator",
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/modules/objectron/calculators:filter_detection_calculator",
],
)
@@ -133,9 +120,7 @@ mediapipe_simple_subgraph(
"//mediapipe/calculators/tflite:tflite_inference_calculator",
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/modules/objectron/calculators:filter_detection_calculator",
],
)
@@ -147,9 +132,19 @@ mediapipe_simple_subgraph(
deps = [
":box_landmark_cpu",
":object_detection_oid_v4_cpu",
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:clip_vector_size_calculator",
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:merge_calculator",
"//mediapipe/calculators/core:previous_loopback_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/modules/objectron/calculators:frame_annotation_to_rect_calculator",
"//mediapipe/modules/objectron/calculators:landmarks_to_frame_annotation_calculator",
"//mediapipe/modules/objectron/calculators:lift_2d_frame_annotation_to_3d_calculator",
],
)
@@ -160,9 +155,18 @@ mediapipe_simple_subgraph(
deps = [
":box_landmark_gpu",
":object_detection_oid_v4_gpu",
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:clip_vector_size_calculator",
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:merge_calculator",
"//mediapipe/calculators/core:previous_loopback_calculator",
"//mediapipe/calculators/image:image_cropping_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/modules/objectron/calculators:frame_annotation_to_rect_calculator",
"//mediapipe/modules/objectron/calculators:landmarks_to_frame_annotation_calculator",
"//mediapipe/modules/objectron/calculators:lift_2d_frame_annotation_to_3d_calculator",
],
)
mediapipe/modules/objectron/box_landmark_cpu.pbtxt
+59 -118
View File
@@ -2,82 +2,75 @@
type: "BoxLandmarkSubgraph"
input_stream: "IMAGE:input_video"
input_stream: "IMAGE:image"
input_stream: "NORM_RECT:box_rect"
input_side_packet: "MODEL:model"
output_stream: "LANDMARKS:box_landmarks_filtered"
output_stream: "NORM_RECT:box_rect_for_next_frame"
output_stream: "PRESENCE:box_presence"
output_stream: "NORM_LANDMARKS:box_landmarks"
# Crops the rectangle that contains a box from the input image.
# Extracts image size from the input images.
node {
calculator: "ImageCroppingCalculator"
input_stream: "IMAGE:input_video"
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:image"
output_stream: "SIZE:image_size"
}
# Expands the rectangle that contain the box so that it's likely to cover the
# entire box.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:box_rect"
output_stream: "IMAGE:box_image"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "box_rect_scaled"
options: {
[mediapipe.ImageCroppingCalculatorOptions.ext] {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 1.5
scale_y: 1.5
square_long: true
}
}
}
# Crops, resizes, and converts the input video into tensor.
# Preserves aspect ratio of the images.
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:image"
input_stream: "NORM_RECT:box_rect_scaled"
output_stream: "TENSORS:image_tensor"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 224
output_tensor_height: 224
keep_aspect_ratio: true
output_tensor_float_range {
min: 0.0
max: 1.0
}
gpu_origin: TOP_LEFT
border_mode: BORDER_REPLICATE
}
}
}
# Transforms the input image to a 224x224 image. To scale the input
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
# resulting in potential letterboxing in the transformed image.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:box_image"
output_stream: "IMAGE:transformed_box_image"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext] {
output_width: 224
output_height: 224
scale_mode: FIT
}
}
}
# Converts the transformed input image into an image tensor stored as a
# TfLiteTensor.
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE:transformed_box_image"
output_stream: "TENSORS:image_tensor"
options: {
[mediapipe.TfLiteConverterCalculatorOptions.ext] {
zero_center: false
}
}
}
# Generates a single side packet containing a TensorFlow Lite op resolver that
# supports custom ops needed by the model used in this graph.
node {
calculator: "TfLiteCustomOpResolverCalculator"
output_side_packet: "opresolver"
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "TfLiteInferenceCalculator"
calculator: "InferenceCalculator"
input_stream: "TENSORS:image_tensor"
output_stream: "TENSORS:output_tensors"
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
input_side_packet: "MODEL:model"
output_stream: "TENSORS:output_tensors"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
use_gpu: false
[mediapipe.InferenceCalculatorOptions.ext] {
delegate { xnnpack {} }
}
}
}
# Splits a vector of tensors into multiple vectors.
node {
calculator: "SplitTfLiteTensorVectorCalculator"
calculator: "SplitTensorVectorCalculator"
input_stream: "output_tensors"
output_stream: "landmark_tensors"
output_stream: "box_flag_tensor"
@@ -92,7 +85,7 @@ node {
# Converts the box-flag tensor into a float that represents the confidence
# score of box presence.
node {
calculator: "TfLiteTensorsToFloatsCalculator"
calculator: "TensorsToFloatsCalculator"
input_stream: "TENSORS:box_flag_tensor"
output_stream: "FLOAT:box_presence_score"
}
@@ -105,19 +98,27 @@ node {
output_stream: "FLAG:box_presence"
options: {
[mediapipe.ThresholdingCalculatorOptions.ext] {
threshold: 0.1
threshold: 0.99
}
}
}
# Drops landmarks tensors if box is not present.
node {
calculator: "GateCalculator"
input_stream: "landmark_tensors"
input_stream: "ALLOW:box_presence"
output_stream: "gated_landmark_tensors"
}
# Decodes the landmark tensors into a list of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TfLiteTensorsToLandmarksCalculator"
input_stream: "TENSORS:landmark_tensors"
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:gated_landmark_tensors"
output_stream: "NORM_LANDMARKS:landmarks"
options: {
[mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 9
input_image_width: 224
input_image_height: 224
@@ -141,66 +142,6 @@ node {
node {
calculator: "LandmarkProjectionCalculator"
input_stream: "NORM_LANDMARKS:scaled_landmarks"
input_stream: "NORM_RECT:box_rect"
input_stream: "NORM_RECT:box_rect_scaled"
output_stream: "NORM_LANDMARKS:box_landmarks"
}
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:input_video"
output_stream: "SIZE:image_size"
}
# Smooth predicted landmarks coordinates.
node {
calculator: "LandmarksSmoothingCalculator"
input_stream: "NORM_LANDMARKS:box_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_FILTERED_LANDMARKS:box_landmarks_filtered"
options: {
[mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
velocity_filter: {
window_size: 10
velocity_scale: 7.5
}
}
}
}
# Convert box landmarks to frame annotation.
node {
calculator: "LandmarksToFrameAnnotationCalculator"
input_stream: "LANDMARKS:box_landmarks_filtered"
output_stream: "FRAME_ANNOTATION:box_annotation"
}
# Lift the 2D landmarks to 3D using EPnP algorithm.
node {
calculator: "Lift2DFrameAnnotationTo3DCalculator"
input_stream: "FRAME_ANNOTATION:box_annotation"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_box"
}
# Get rotated rectangle from lifted box.
node {
calculator: "FrameAnnotationToRectCalculator"
input_stream: "FRAME_ANNOTATION:lifted_box"
output_stream: "NORM_RECT:rect_from_box"
}
# Expands the box rectangle so that in the next video frame it's likely to
# still contain the box even with some motion.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:rect_from_box"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "box_rect_for_next_frame"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 1.5
scale_y: 1.5
square_long: true
}
}
}
mediapipe/modules/objectron/box_landmark_gpu.pbtxt
+60 -118
View File
@@ -2,81 +2,75 @@
type: "BoxLandmarkSubgraph"
input_stream: "IMAGE:input_video"
input_stream: "IMAGE:image"
input_stream: "NORM_RECT:box_rect"
output_stream: "FRAME_ANNOTATION:lifted_box"
output_stream: "NORM_RECT:box_rect_for_next_frame"
output_stream: "PRESENCE:box_presence"
output_stream: "NORM_LANDMARKS:box_landmarks"
# Crops the rectangle that contains a box from the input image.
# Extracts image size from the input images.
node {
calculator: "ImageCroppingCalculator"
input_stream: "IMAGE_GPU:input_video"
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "SIZE:image_size"
}
# Expands the rectangle that contain the box so that it's likely to cover the
# entire box.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:box_rect"
output_stream: "IMAGE_GPU:box_image"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "box_rect_scaled"
options: {
[mediapipe.ImageCroppingCalculatorOptions.ext] {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 1.5
scale_y: 1.5
square_long: true
}
}
}
# Crops, resizes, and converts the input video into tensor.
# Preserves aspect ratio of the images.
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE_GPU:image"
input_stream: "NORM_RECT:box_rect_scaled"
output_stream: "TENSORS:image_tensor"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 224
output_tensor_height: 224
keep_aspect_ratio: true
output_tensor_float_range {
min: 0.0
max: 1.0
}
gpu_origin: TOP_LEFT
border_mode: BORDER_REPLICATE
}
}
}
# Transforms the input image on GPU to a 224x224 image. To scale the input
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
# resulting in potential letterboxing in the transformed image.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:box_image"
output_stream: "IMAGE_GPU:transformed_box_image"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext] {
output_width: 224
output_height: 224
scale_mode: FIT
}
}
}
# Converts the transformed input image on GPU into an image tensor stored as a
# TfLiteTensor.
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE_GPU:transformed_box_image"
output_stream: "TENSORS_GPU:image_tensor"
options: {
[mediapipe.TfLiteConverterCalculatorOptions.ext] {
zero_center: false
}
}
}
# Generates a single side packet containing a TensorFlow Lite op resolver that
# supports custom ops needed by the model used in this graph.
node {
calculator: "TfLiteCustomOpResolverCalculator"
output_side_packet: "opresolver"
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS_GPU:image_tensor"
calculator: "InferenceCalculator"
input_stream: "TENSORS:image_tensor"
output_stream: "TENSORS:output_tensors"
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "object_detection_3d.tflite"
use_gpu: true
delegate { gpu {} }
}
}
}
# Splits a vector of tensors into multiple vectors.
# Splits a vector of tensors to multiple vectors according to the ranges
# specified in option.
node {
calculator: "SplitTfLiteTensorVectorCalculator"
calculator: "SplitTensorVectorCalculator"
input_stream: "output_tensors"
output_stream: "landmark_tensors"
output_stream: "box_flag_tensor"
@@ -91,7 +85,7 @@ node {
# Converts the box-flag tensor into a float that represents the confidence
# score of box presence.
node {
calculator: "TfLiteTensorsToFloatsCalculator"
calculator: "TensorsToFloatsCalculator"
input_stream: "TENSORS:box_flag_tensor"
output_stream: "FLOAT:box_presence_score"
}
@@ -109,14 +103,22 @@ node {
}
}
# Drops landmarks tensors if box is not present.
node {
calculator: "GateCalculator"
input_stream: "landmark_tensors"
input_stream: "ALLOW:box_presence"
output_stream: "gated_landmark_tensors"
}
# Decodes the landmark tensors into a list of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TfLiteTensorsToLandmarksCalculator"
input_stream: "TENSORS:landmark_tensors"
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:gated_landmark_tensors"
output_stream: "NORM_LANDMARKS:landmarks"
options: {
[mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 9
input_image_width: 224
input_image_height: 224
@@ -140,66 +142,6 @@ node {
node {
calculator: "LandmarkProjectionCalculator"
input_stream: "NORM_LANDMARKS:scaled_landmarks"
input_stream: "NORM_RECT:box_rect"
input_stream: "NORM_RECT:box_rect_scaled"
output_stream: "NORM_LANDMARKS:box_landmarks"
}
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_video"
output_stream: "SIZE:image_size"
}
# Smooth predicted landmarks coordinates.
node {
calculator: "LandmarksSmoothingCalculator"
input_stream: "NORM_LANDMARKS:box_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_FILTERED_LANDMARKS:box_landmarks_filtered"
options: {
[mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
velocity_filter: {
window_size: 10
velocity_scale: 7.5
}
}
}
}
# Convert box landmarks to frame annotation.
node {
calculator: "LandmarksToFrameAnnotationCalculator"
input_stream: "LANDMARKS:box_landmarks_filtered"
output_stream: "FRAME_ANNOTATION:box_annotation"
}
# Lift the 2D landmarks to 3D using EPnP algorithm.
node {
calculator: "Lift2DFrameAnnotationTo3DCalculator"
input_stream: "FRAME_ANNOTATION:box_annotation"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_box"
}
# Get rotated rectangle from lifted box.
node {
calculator: "FrameAnnotationToRectCalculator"
input_stream: "FRAME_ANNOTATION:lifted_box"
output_stream: "NORM_RECT:rect_from_box"
}
# Expands the box rectangle so that in the next video frame it's likely to
# still contain the box even with some motion.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:rect_from_box"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "box_rect_for_next_frame"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 1.5
scale_y: 1.5
square_long: true
}
}
}
mediapipe/modules/objectron/calculators/frame_annotation_to_rect_calculator.cc
+46 -43
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
#include <cmath>
#include <vector>
#include "Eigen/Dense"
#include "absl/memory/memory.h"
@@ -32,7 +33,7 @@ using Eigen::Vector3f;
namespace {
constexpr char kInputFrameAnnotationTag[] = "FRAME_ANNOTATION";
constexpr char kOutputNormRectTag[] = "NORM_RECT";
constexpr char kOutputNormRectsTag[] = "NORM_RECTS";
} // namespace
@@ -47,14 +48,14 @@ class FrameAnnotationToRectCalculator : public CalculatorBase {
TOP_VIEW_OFF,
};
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
private:
void AnnotationToRect(const FrameAnnotation& annotation,
NormalizedRect* rect);
float RotationAngleFromAnnotation(const FrameAnnotation& annotation);
void AddAnnotationToRect(const ObjectAnnotation& annotation,
std::vector<NormalizedRect>* rect);
float RotationAngleFromAnnotation(const ObjectAnnotation& annotation);
float RotationAngleFromPose(const Matrix3fRM& rotation,
const Vector3f& translation, const Vector3f& vec);
@@ -64,17 +65,7 @@ class FrameAnnotationToRectCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(FrameAnnotationToRectCalculator);
::mediapipe::Status FrameAnnotationToRectCalculator::Open(
CalculatorContext* cc) {
status_ = TOP_VIEW_OFF;
const auto& options = cc->Options<FrameAnnotationToRectCalculatorOptions>();
off_threshold_ = options.off_threshold();
on_threshold_ = options.on_threshold();
RET_CHECK(off_threshold_ <= on_threshold_);
return ::mediapipe::OkStatus();
}
::mediapipe::Status FrameAnnotationToRectCalculator::GetContract(
mediapipe::Status FrameAnnotationToRectCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -83,57 +74,69 @@ REGISTER_CALCULATOR(FrameAnnotationToRectCalculator);
cc->Inputs().Tag(kInputFrameAnnotationTag).Set<FrameAnnotation>();
}
if (cc->Outputs().HasTag(kOutputNormRectTag)) {
cc->Outputs().Tag(kOutputNormRectTag).Set<NormalizedRect>();
if (cc->Outputs().HasTag(kOutputNormRectsTag)) {
cc->Outputs().Tag(kOutputNormRectsTag).Set<std::vector<NormalizedRect>>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status FrameAnnotationToRectCalculator::Process(
mediapipe::Status FrameAnnotationToRectCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
status_ = TOP_VIEW_OFF;
const auto& options = cc->Options<FrameAnnotationToRectCalculatorOptions>();
off_threshold_ = options.off_threshold();
on_threshold_ = options.on_threshold();
RET_CHECK(off_threshold_ <= on_threshold_);
return mediapipe::OkStatus();
}
mediapipe::Status FrameAnnotationToRectCalculator::Process(
CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputFrameAnnotationTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
auto output_rects = absl::make_unique<std::vector<NormalizedRect>>();
const auto& frame_annotation =
cc->Inputs().Tag(kInputFrameAnnotationTag).Get<FrameAnnotation>();
for (const auto& object_annotation : frame_annotation.annotations()) {
AddAnnotationToRect(object_annotation, output_rects.get());
}
auto output_rect = absl::make_unique<NormalizedRect>();
AnnotationToRect(
cc->Inputs().Tag(kInputFrameAnnotationTag).Get<FrameAnnotation>(),
output_rect.get());
// Output
// Output.
cc->Outputs()
.Tag(kOutputNormRectTag)
.Add(output_rect.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
.Tag(kOutputNormRectsTag)
.Add(output_rects.release(), cc->InputTimestamp());
return mediapipe::OkStatus();
}
void FrameAnnotationToRectCalculator::AnnotationToRect(
const FrameAnnotation& annotation, NormalizedRect* rect) {
void FrameAnnotationToRectCalculator::AddAnnotationToRect(
const ObjectAnnotation& annotation, std::vector<NormalizedRect>* rects) {
float x_min = std::numeric_limits<float>::max();
float x_max = std::numeric_limits<float>::min();
float y_min = std::numeric_limits<float>::max();
float y_max = std::numeric_limits<float>::min();
const auto& object = annotation.annotations(0);
for (const auto& keypoint : object.keypoints()) {
for (const auto& keypoint : annotation.keypoints()) {
const auto& point_2d = keypoint.point_2d();
x_min = std::min(x_min, point_2d.x());
x_max = std::max(x_max, point_2d.x());
y_min = std::min(y_min, point_2d.y());
y_max = std::max(y_max, point_2d.y());
}
rect->set_x_center((x_min + x_max) / 2);
rect->set_y_center((y_min + y_max) / 2);
rect->set_width(x_max - x_min);
rect->set_height(y_max - y_min);
rect->set_rotation(RotationAngleFromAnnotation(annotation));
NormalizedRect new_rect;
new_rect.set_x_center((x_min + x_max) / 2);
new_rect.set_y_center((y_min + y_max) / 2);
new_rect.set_width(x_max - x_min);
new_rect.set_height(y_max - y_min);
new_rect.set_rotation(RotationAngleFromAnnotation(annotation));
rects->push_back(new_rect);
}
float FrameAnnotationToRectCalculator::RotationAngleFromAnnotation(
const FrameAnnotation& annotation) {
const auto& object = annotation.annotations(0);
const ObjectAnnotation& annotation) {
Box box("category");
std::vector<Vector3f> vertices_3d;
std::vector<Vector2f> vertices_2d;
for (const auto& keypoint : object.keypoints()) {
for (const auto& keypoint : annotation.keypoints()) {
const auto& point_3d = keypoint.point_3d();
const auto& point_2d = keypoint.point_2d();
vertices_3d.emplace_back(
mediapipe/modules/objectron/calculators/landmarks_to_frame_annotation_calculator.cc
+53 -16
View File
@@ -23,6 +23,7 @@ namespace mediapipe {
namespace {
constexpr char kInputLandmarksTag[] = "LANDMARKS";
constexpr char kInputMultiLandmarksTag[] = "MULTI_LANDMARKS";
constexpr char kOutputFrameAnnotationTag[] = "FRAME_ANNOTATION";
} // namespace
@@ -30,12 +31,17 @@ constexpr char kOutputFrameAnnotationTag[] = "FRAME_ANNOTATION";
// A calculator that converts NormalizedLandmarkList to FrameAnnotation proto.
class LandmarksToFrameAnnotationCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Process(CalculatorContext* cc) override;
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
private:
void AddLandmarksToFrameAnnotation(const NormalizedLandmarkList& landmarks,
FrameAnnotation* frame_annotation);
};
REGISTER_CALCULATOR(LandmarksToFrameAnnotationCalculator);
::mediapipe::Status LandmarksToFrameAnnotationCalculator::GetContract(
mediapipe::Status LandmarksToFrameAnnotationCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -43,34 +49,65 @@ REGISTER_CALCULATOR(LandmarksToFrameAnnotationCalculator);
if (cc->Inputs().HasTag(kInputLandmarksTag)) {
cc->Inputs().Tag(kInputLandmarksTag).Set<NormalizedLandmarkList>();
}
if (cc->Inputs().HasTag(kInputMultiLandmarksTag)) {
cc->Inputs()
.Tag(kInputMultiLandmarksTag)
.Set<std::vector<NormalizedLandmarkList>>();
}
if (cc->Outputs().HasTag(kOutputFrameAnnotationTag)) {
cc->Outputs().Tag(kOutputFrameAnnotationTag).Set<FrameAnnotation>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status LandmarksToFrameAnnotationCalculator::Process(
mediapipe::Status LandmarksToFrameAnnotationCalculator::Open(
CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
return mediapipe::OkStatus();
}
mediapipe::Status LandmarksToFrameAnnotationCalculator::Process(
CalculatorContext* cc) {
auto frame_annotation = absl::make_unique<FrameAnnotation>();
auto* box_annotation = frame_annotation->add_annotations();
const auto& landmarks =
cc->Inputs().Tag(kInputLandmarksTag).Get<NormalizedLandmarkList>();
RET_CHECK_GT(landmarks.landmark_size(), 0)
<< "Input landmark vector is empty.";
for (int i = 0; i < landmarks.landmark_size(); ++i) {
auto* point2d = box_annotation->add_keypoints()->mutable_point_2d();
point2d->set_x(landmarks.landmark(i).x());
point2d->set_y(landmarks.landmark(i).y());
// Handle the case when input has only one NormalizedLandmarkList.
if (cc->Inputs().HasTag(kInputLandmarksTag) &&
!cc->Inputs().Tag(kInputLandmarksTag).IsEmpty()) {
const auto& landmarks =
cc->Inputs().Tag(kInputMultiLandmarksTag).Get<NormalizedLandmarkList>();
AddLandmarksToFrameAnnotation(landmarks, frame_annotation.get());
}
// Handle the case when input has muliple NormalizedLandmarkList.
if (cc->Inputs().HasTag(kInputMultiLandmarksTag) &&
!cc->Inputs().Tag(kInputMultiLandmarksTag).IsEmpty()) {
const auto& landmarks_list =
cc->Inputs()
.Tag(kInputMultiLandmarksTag)
.Get<std::vector<NormalizedLandmarkList>>();
for (const auto& landmarks : landmarks_list) {
AddLandmarksToFrameAnnotation(landmarks, frame_annotation.get());
}
}
// Output
if (cc->Outputs().HasTag(kOutputFrameAnnotationTag)) {
cc->Outputs()
.Tag(kOutputFrameAnnotationTag)
.Add(frame_annotation.release(), cc->InputTimestamp());
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void LandmarksToFrameAnnotationCalculator::AddLandmarksToFrameAnnotation(
const NormalizedLandmarkList& landmarks,
FrameAnnotation* frame_annotation) {
auto* new_annotation = frame_annotation->add_annotations();
for (const auto& landmark : landmarks.landmark()) {
auto* point2d = new_annotation->add_keypoints()->mutable_point_2d();
point2d->set_x(landmark.x());
point2d->set_y(landmark.y());
}
}
} // namespace mediapipe
mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.cc
+23 -23
View File
@@ -55,16 +55,16 @@ namespace mediapipe {
// }
class Lift2DFrameAnnotationTo3DCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
::mediapipe::Status ProcessCPU(CalculatorContext* cc,
FrameAnnotation* output_objects);
::mediapipe::Status LoadOptions(CalculatorContext* cc);
mediapipe::Status ProcessCPU(CalculatorContext* cc,
FrameAnnotation* output_objects);
mediapipe::Status LoadOptions(CalculatorContext* cc);
// Increment and assign object ID for each detected object.
// In a single MediaPipe session, the IDs are unique.
@@ -73,23 +73,24 @@ class Lift2DFrameAnnotationTo3DCalculator : public CalculatorBase {
void AssignObjectIdAndTimestamp(int64 timestamp_us,
FrameAnnotation* annotation);
std::unique_ptr<Decoder> decoder_;
::mediapipe::Lift2DFrameAnnotationTo3DCalculatorOptions options_;
Lift2DFrameAnnotationTo3DCalculatorOptions options_;
Eigen::Matrix<float, 4, 4, Eigen::RowMajor> projection_matrix_;
};
REGISTER_CALCULATOR(Lift2DFrameAnnotationTo3DCalculator);
::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::GetContract(
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag(kInputStreamTag));
RET_CHECK(cc->Outputs().HasTag(kOutputStreamTag));
cc->Inputs().Tag(kInputStreamTag).Set<FrameAnnotation>();
cc->Outputs().Tag(kOutputStreamTag).Set<FrameAnnotation>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Open(
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Open(
CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
MP_RETURN_IF_ERROR(LoadOptions(cc));
// clang-format off
projection_matrix_ <<
@@ -101,13 +102,13 @@ REGISTER_CALCULATOR(Lift2DFrameAnnotationTo3DCalculator);
decoder_ = absl::make_unique<Decoder>(
BeliefDecoderConfig(options_.decoder_config()));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Process(
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Process(
CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
auto output_objects = absl::make_unique<FrameAnnotation>();
@@ -121,10 +122,10 @@ REGISTER_CALCULATOR(Lift2DFrameAnnotationTo3DCalculator);
.Add(output_objects.release(), cc->InputTimestamp());
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::ProcessCPU(
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::ProcessCPU(
CalculatorContext* cc, FrameAnnotation* output_objects) {
const auto& input_frame_annotations =
cc->Inputs().Tag(kInputStreamTag).Get<FrameAnnotation>();
@@ -140,21 +141,20 @@ REGISTER_CALCULATOR(Lift2DFrameAnnotationTo3DCalculator);
AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(),
output_objects);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Close(
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Close(
CalculatorContext* cc) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::LoadOptions(
mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::LoadOptions(
CalculatorContext* cc) {
// Get calculator options specified in the graph.
options_ =
cc->Options<::mediapipe::Lift2DFrameAnnotationTo3DCalculatorOptions>();
options_ = cc->Options<Lift2DFrameAnnotationTo3DCalculatorOptions>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void Lift2DFrameAnnotationTo3DCalculator::AssignObjectIdAndTimestamp(
mediapipe/modules/objectron/object_detection_oid_v4_cpu.pbtxt
+21 -60
View File
@@ -4,9 +4,9 @@ type: "ObjectDetectionOidV4Subgraph"
input_stream: "IMAGE:input_video"
input_side_packet: "LABELS_CSV:allowed_labels"
output_stream: "NORM_RECT:box_rect_from_object_detections"
output_stream: "DETECTIONS:detections"
# Transforms the input image on GPU to a 300x300 image. To scale the image, by
# Transforms the input image on CPU to a 300x300 image. To scale the image, by
# default it uses the STRETCH scale mode that maps the entire input image to the
# entire transformed image. As a result, image aspect ratio may be changed and
# objects in the image may be deformed (stretched or squeezed), but the object
@@ -23,7 +23,7 @@ node: {
}
}
# Converts the transformed input image on GPU into an image tensor stored as a
# Converts the transformed input image on CPU into an image tensor stored as a
# TfLiteTensor.
node {
calculator: "TfLiteConverterCalculator"
@@ -31,7 +31,7 @@ node {
output_stream: "TENSORS:image_tensor"
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
@@ -82,7 +82,7 @@ node {
calculator: "TfLiteTensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:detections"
output_stream: "DETECTIONS:all_detections"
options: {
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
num_classes: 195
@@ -95,22 +95,7 @@ node {
y_scale: 10.0
h_scale: 5.0
w_scale: 5.0
min_score_thresh: 0.6
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "detections"
output_stream: "suppressed_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.4
max_num_detections: 1
overlap_type: INTERSECTION_OVER_UNION
return_empty_detections: true
min_score_thresh: 0.5
}
}
}
@@ -119,7 +104,7 @@ node {
# provided in the label_map_path option.
node {
calculator: "DetectionLabelIdToTextCalculator"
input_stream: "suppressed_detections"
input_stream: "all_detections"
output_stream: "labeled_detections"
options: {
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
@@ -128,50 +113,26 @@ node {
}
}
# Filters the detections to only those with valid scores
# for the specified allowed labels.
node {
calculator: "FilterDetectionCalculator"
input_stream: "DETECTIONS:labeled_detections"
output_stream: "DETECTIONS:filtered_detections"
input_side_packet: "LABELS_CSV:allowed_labels"
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "filtered_detections"
output_stream: "detections"
options: {
[mediapipe.FilterDetectionCalculatorOptions.ext]: {
min_score: 0.4
}
}
}
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:input_video"
output_stream: "SIZE:image_size"
}
# Converts results of box detection into a rectangle (normalized by image size)
# that encloses the box.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:box_rect"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
output_zero_rect_for_empty_detections: true
}
}
}
# Expands the rectangle that contains the box so that it's likely to cover the
# entire box.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:box_rect"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "box_rect_from_object_detections"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 1.5
scale_y: 1.5
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.5
max_num_detections: 100
overlap_type: INTERSECTION_OVER_UNION
return_empty_detections: true
}
}
}
mediapipe/modules/objectron/object_detection_oid_v4_gpu.pbtxt
+20 -59
View File
@@ -4,7 +4,7 @@ type: "ObjectDetectionOidV4Subgraph"
input_stream: "IMAGE_GPU:input_video"
input_side_packet: "LABELS_CSV:allowed_labels"
output_stream: "NORM_RECT:box_rect_from_object_detections"
output_stream: "DETECTIONS:detections"
# Transforms the input image on GPU to a 300x300 image. To scale the image, by
# default it uses the STRETCH scale mode that maps the entire input image to the
@@ -40,7 +40,7 @@ node {
output_stream: "TENSORS_GPU:detection_tensors"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "mediapipe/models/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
model_path: "object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
}
}
}
@@ -82,7 +82,7 @@ node {
calculator: "TfLiteTensorsToDetectionsCalculator"
input_stream: "TENSORS_GPU:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:detections"
output_stream: "DETECTIONS:all_detections"
options: {
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
num_classes: 195
@@ -95,22 +95,7 @@ node {
y_scale: 10.0
h_scale: 5.0
w_scale: 5.0
min_score_thresh: 0.6
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "detections"
output_stream: "suppressed_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.4
max_num_detections: 1
overlap_type: INTERSECTION_OVER_UNION
return_empty_detections: true
min_score_thresh: 0.5
}
}
}
@@ -119,59 +104,35 @@ node {
# provided in the label_map_path option.
node {
calculator: "DetectionLabelIdToTextCalculator"
input_stream: "suppressed_detections"
input_stream: "all_detections"
output_stream: "labeled_detections"
options: {
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
label_map_path: "mediapipe/models/object_detection_oidv4_labelmap.pbtxt"
label_map_path: "object_detection_oidv4_labelmap.pbtxt"
}
}
}
# Filters the detections to only those with valid scores
# for the specified allowed labels.
node {
calculator: "FilterDetectionCalculator"
input_stream: "DETECTIONS:labeled_detections"
output_stream: "DETECTIONS:filtered_detections"
input_side_packet: "LABELS_CSV:allowed_labels"
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "filtered_detections"
output_stream: "detections"
options: {
[mediapipe.FilterDetectionCalculatorOptions.ext]: {
min_score: 0.4
}
}
}
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_video"
output_stream: "SIZE:image_size"
}
# Converts results of box detection into a rectangle (normalized by image size)
# that encloses the box.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:box_rect"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
output_zero_rect_for_empty_detections: true
}
}
}
# Expands the rectangle that contains the box so that it's likely to cover the
# entire box.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:box_rect"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "box_rect_from_object_detections"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 1.5
scale_y: 1.5
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.5
max_num_detections: 100
overlap_type: INTERSECTION_OVER_UNION
return_empty_detections: true
}
}
}
mediapipe/modules/objectron/objectron_cpu.pbtxt
+134 -47
View File
@@ -4,6 +4,8 @@ input_stream: "IMAGE:input_video"
input_side_packet: "MODEL:box_landmark_model"
# Allowed category labels, e.g. Footwear, Coffee cup, Mug, Chair, Camera
input_side_packet: "LABELS_CSV:allowed_labels"
# Max number of objects to detect/track. (int)
input_side_packet: "MAX_NUM_OBJECTS:max_num_objects"
# Bounding box landmarks topology definition.
# The numbers are indices in the box_landmarks list.
#
@@ -22,36 +24,47 @@ input_side_packet: "LABELS_CSV:allowed_labels"
# \+ \+
# 2 + + + + + + + + 6
#
output_stream: "LANDMARKS:box_landmarks"
# Crop rectangle derived from bounding box landmarks.
output_stream: "NORM_RECT:box_rect"
output_stream: "MULTI_LANDMARKS:multi_box_landmarks"
# Crop rectangles derived from bounding box landmarks.
output_stream: "NORM_RECTS:multi_box_rects"
# Caches a box-presence decision fed back from boxLandmarkSubgraph, and upon
# the arrival of the next input image sends out the cached decision with the
# timestamp replaced by that of the input image, essentially generating a packet
# that carries the previous box-presence decision. Note that upon the arrival
# of the very first input image, an empty packet is sent out to jump start the
# feedback loop.
# Defines whether landmarks from the previous video frame should be used to help
# predict landmarks on the current video frame.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:input_video"
input_stream: "LOOP:box_presence"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
name: "ConstantSidePacketCalculator"
calculator: "ConstantSidePacketCalculator"
output_side_packet: "PACKET:use_prev_landmarks"
options: {
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
packet { bool_value: true }
}
}
output_stream: "PREV_LOOP:prev_box_presence"
}
# Drops the incoming image if boxLandmarkSubgraph was able to identify box
node {
calculator: "GateCalculator"
input_side_packet: "ALLOW:use_prev_landmarks"
input_stream: "prev_box_rects_from_landmarks"
output_stream: "gated_prev_box_rects_from_landmarks"
}
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided max_num_objects.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:gated_prev_box_rects_from_landmarks"
input_side_packet: "max_num_objects"
output_stream: "prev_has_enough_objects"
}
# Drops the incoming image if BoxLandmarkSubgraph was able to identify box
# presence in the previous image. Otherwise, passes the incoming image through
# to trigger a new round of box detection in boxDetectionSubgraph.
# to trigger a new round of box detection in ObjectDetectionOidV4Subgraph.
node {
calculator: "GateCalculator"
input_stream: "input_video"
input_stream: "DISALLOW:prev_box_presence"
output_stream: "box_detection_input_video"
input_stream: "DISALLOW:prev_has_enough_objects"
output_stream: "detection_input_video"
options: {
[mediapipe.GateCalculatorOptions.ext] {
@@ -60,23 +73,112 @@ node {
}
}
# Subgraph that detections boxs (see object_detection_oid_v4_cpu.pbtxt).
# Subgraph that performs 2D object detection.
node {
calculator: "ObjectDetectionOidV4Subgraph"
input_stream: "IMAGE:box_detection_input_video"
input_stream: "IMAGE:detection_input_video"
input_side_packet: "LABELS_CSV:allowed_labels"
output_stream: "NORM_RECT:box_rect_from_object_detections"
output_stream: "DETECTIONS:raw_detections"
}
# Subgraph that localizes box landmarks (see box_landmark_gpu.pbtxt).
# Makes sure there are no more detections than provided max_num_objects.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "raw_detections"
output_stream: "detections"
input_side_packet: "max_num_objects"
}
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:input_video"
output_stream: "SIZE:image_size"
}
# Converts results of box detection into rectangles (normalized by image size)
# that encloses the box.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTIONS:detections"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECTS:box_rects_from_detections"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
output_zero_rect_for_empty_detections: false
}
}
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on object detections from the current image. This
# calculator ensures that the output box_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "box_rects_from_detections"
input_stream: "gated_prev_box_rects_from_landmarks"
output_stream: "multi_box_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.2
}
}
}
# Outputs each element of box_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_box_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:multi_box_rects"
input_stream: "CLONE:input_video"
output_stream: "ITEM:single_box_rect"
output_stream: "CLONE:landmarks_input_video"
output_stream: "BATCH_END:box_rects_timestamp"
}
# Subgraph that localizes box landmarks.
node {
calculator: "BoxLandmarkSubgraph"
input_stream: "IMAGE:input_video"
input_stream: "NORM_RECT:box_rect"
input_stream: "IMAGE:landmarks_input_video"
input_side_packet: "MODEL:box_landmark_model"
output_stream: "LANDMARKS:box_landmarks"
output_stream: "NORM_RECT:box_rect_from_landmarks"
output_stream: "PRESENCE:box_presence"
input_stream: "NORM_RECT:single_box_rect"
output_stream: "NORM_LANDMARKS:single_box_landmarks"
}
# Collects a set of landmarks for each hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:single_box_landmarks"
input_stream: "BATCH_END:box_rects_timestamp"
output_stream: "ITERABLE:multi_box_landmarks"
}
# Convert box landmarks to frame annotations.
node {
calculator: "LandmarksToFrameAnnotationCalculator"
input_stream: "MULTI_LANDMARKS:multi_box_landmarks"
output_stream: "FRAME_ANNOTATION:box_annotations"
}
# Lift the 2D landmarks to 3D using EPnP algorithm.
node {
calculator: "Lift2DFrameAnnotationTo3DCalculator"
input_stream: "FRAME_ANNOTATION:box_annotations"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_objects"
}
# Get rotated rectangle from lifted box.
node {
calculator: "FrameAnnotationToRectCalculator"
input_stream: "FRAME_ANNOTATION:lifted_objects"
output_stream: "NORM_RECTS:box_rects_from_landmarks"
}
# Caches a box rectangle fed back from boxLandmarkSubgraph, and upon the
@@ -88,25 +190,10 @@ node {
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:input_video"
input_stream: "LOOP:box_rect_from_landmarks"
input_stream: "LOOP:box_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_box_rect_from_landmarks"
}
# Merges a stream of box rectangles generated by ObjectDetectionSubgraph and that
# generated by BoxLandmarkSubgraph into a single output stream by selecting
# between one of the two streams. The former is selected if the incoming packet
# is not empty, i.e., box detection is performed on the current image by
# BoxDetectionSubgraph (because BoxLandmarkSubgraph could not identify box
# presence in the previous image). Otherwise, the latter is selected, which is
# never empty because BoxLandmarkSubgraphs processes all images (that went
# through FlowLimiterCaculator).
node {
calculator: "MergeCalculator"
input_stream: "box_rect_from_object_detections"
input_stream: "prev_box_rect_from_landmarks"
output_stream: "box_rect"
output_stream: "PREV_LOOP:prev_box_rects_from_landmarks"
}
mediapipe/modules/objectron/objectron_gpu.pbtxt
+129 -40
View File
@@ -5,33 +5,48 @@
input_stream: "IMAGE_GPU:input_video"
# Allowed category labels, e.g. Footwear, Coffee cup, Mug, Chair, Camera
input_side_packet: "LABELS_CSV:allowed_labels"
# Max number of objects to detect/track. (int)
input_side_packet: "MAX_NUM_OBJECTS:max_num_objects"
# Collection of detected 3D objects, represented as a FrameAnnotation.
output_stream: "FRAME_ANNOTATION:lifted_objects"
# Caches a box-presence decision fed back from boxLandmarkSubgraph, and upon
# the arrival of the next input image sends out the cached decision with the
# timestamp replaced by that of the input image, essentially generating a packet
# that carries the previous box-presence decision. Note that upon the arrival
# of the very first input image, an empty packet is sent out to jump start the
# feedback loop.
# Defines whether landmarks from the previous video frame should be used to help
# predict landmarks on the current video frame.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:input_video"
input_stream: "LOOP:box_presence"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
name: "ConstantSidePacketCalculator"
calculator: "ConstantSidePacketCalculator"
output_side_packet: "PACKET:use_prev_landmarks"
options: {
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
packet { bool_value: true }
}
}
output_stream: "PREV_LOOP:prev_box_presence"
}
# Drops the incoming image if boxLandmarkSubgraph was able to identify box
node {
calculator: "GateCalculator"
input_side_packet: "ALLOW:use_prev_landmarks"
input_stream: "prev_box_rects_from_landmarks"
output_stream: "gated_prev_box_rects_from_landmarks"
}
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided max_num_objects.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:gated_prev_box_rects_from_landmarks"
input_side_packet: "max_num_objects"
output_stream: "prev_has_enough_objects"
}
# Drops the incoming image if BoxLandmarkSubgraph was able to identify box
# presence in the previous image. Otherwise, passes the incoming image through
# to trigger a new round of box detection in boxDetectionSubgraph.
# to trigger a new round of box detection in ObjectDetectionOidV4Subgraph.
node {
calculator: "GateCalculator"
input_stream: "input_video"
input_stream: "DISALLOW:prev_box_presence"
input_stream: "DISALLOW:prev_has_enough_objects"
output_stream: "detection_input_video"
options: {
@@ -46,17 +61,106 @@ node {
calculator: "ObjectDetectionOidV4Subgraph"
input_stream: "IMAGE_GPU:detection_input_video"
input_side_packet: "LABELS_CSV:allowed_labels"
output_stream: "NORM_RECT:box_rect_from_object_detections"
output_stream: "DETECTIONS:raw_detections"
}
# Makes sure there are no more detections than provided max_num_objects.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "raw_detections"
output_stream: "detections"
input_side_packet: "max_num_objects"
}
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_video"
output_stream: "SIZE:image_size"
}
# Converts results of box detection into rectangles (normalized by image size)
# that encloses the box.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTIONS:detections"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECTS:box_rects_from_detections"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
output_zero_rect_for_empty_detections: false
}
}
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on object detections from the current image. This
# calculator ensures that the output box_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "box_rects_from_detections"
input_stream: "gated_prev_box_rects_from_landmarks"
output_stream: "box_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.2
}
}
}
# Outputs each element of box_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_box_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:box_rects"
input_stream: "CLONE:input_video"
output_stream: "ITEM:single_box_rect"
output_stream: "CLONE:landmarks_input_video"
output_stream: "BATCH_END:box_rects_timestamp"
}
# Subgraph that localizes box landmarks.
node {
calculator: "BoxLandmarkSubgraph"
input_stream: "IMAGE:input_video"
input_stream: "NORM_RECT:box_rect"
output_stream: "FRAME_ANNOTATION:lifted_objects"
output_stream: "NORM_RECT:box_rect_from_landmarks"
output_stream: "PRESENCE:box_presence"
input_stream: "IMAGE:landmarks_input_video"
input_stream: "NORM_RECT:single_box_rect"
output_stream: "NORM_LANDMARKS:single_box_landmarks"
}
# Collects a set of landmarks for each hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:single_box_landmarks"
input_stream: "BATCH_END:box_rects_timestamp"
output_stream: "ITERABLE:multi_box_landmarks"
}
# Convert box landmarks to frame annotations.
node {
calculator: "LandmarksToFrameAnnotationCalculator"
input_stream: "MULTI_LANDMARKS:multi_box_landmarks"
output_stream: "FRAME_ANNOTATION:box_annotations"
}
# Lift the 2D landmarks to 3D using EPnP algorithm.
node {
calculator: "Lift2DFrameAnnotationTo3DCalculator"
input_stream: "FRAME_ANNOTATION:box_annotations"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_objects"
}
# Get rotated rectangle from lifted box.
node {
calculator: "FrameAnnotationToRectCalculator"
input_stream: "FRAME_ANNOTATION:lifted_objects"
output_stream: "NORM_RECTS:box_rects_from_landmarks"
}
# Caches a box rectangle fed back from boxLandmarkSubgraph, and upon the
@@ -68,25 +172,10 @@ node {
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:input_video"
input_stream: "LOOP:box_rect_from_landmarks"
input_stream: "LOOP:box_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_box_rect_from_landmarks"
}
# Merges a stream of box rectangles generated by boxDetectionSubgraph and that
# generated by boxLandmarkSubgraph into a single output stream by selecting
# between one of the two streams. The former is selected if the incoming packet
# is not empty, i.e., box detection is performed on the current image by
# boxDetectionSubgraph (because boxLandmarkSubgraph could not identify box
# presence in the previous image). Otherwise, the latter is selected, which is
# never empty because boxLandmarkSubgraphs processes all images (that went
# through FlowLimiterCaculator).
node {
calculator: "MergeCalculator"
input_stream: "box_rect_from_object_detections"
input_stream: "prev_box_rect_from_landmarks"
output_stream: "box_rect"
output_stream: "PREV_LOOP:prev_box_rects_from_landmarks"
}
mediapipe/modules/pose_landmark/pose_landmark_model_loader.pbtxt
+5
View File
@@ -46,6 +46,11 @@ node {
calculator: "LocalFileContentsCalculator"
input_side_packet: "FILE_PATH:model_path"
output_side_packet: "CONTENTS:model_blob"
options: {
[mediapipe.LocalFileContentsCalculatorOptions.ext]: {
read_as_binary: true
}
}
}
# Converts the input blob into a TF Lite model.