Project import generated by Copybara.
GitOrigin-RevId: f72a0f86c2c2acdb1920973c718a9e26ed3ec4b6
This commit is contained in:
MediaPipe Team
@@ -0,0 +1,356 @@
|
||||
---
|
||||
layout: default
|
||||
title: AutoFlip (Saliency-aware Video Cropping)
|
||||
parent: Solutions
|
||||
nav_order: 9
|
||||
---
|
||||
|
||||
# AutoFlip: Saliency-aware Video Cropping
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
AutoFlip is an automatic video cropping pipeline built on top of MediaPipe. This
|
||||
example focuses on demonstrating how to use AutoFlip to convert an input video
|
||||
to arbitrary aspect ratios.
|
||||
|
||||
For overall context on AutoFlip, please read this
|
||||
[Google AI Blog](https://ai.googleblog.com/2020/02/autoflip-open-source-framework-for.html).
|
||||
|
||||
|
||||
|
||||
## Building
|
||||
|
||||
Run the following command to build the AutoFlip pipeline:
|
||||
|
||||
Note: AutoFlip currently only works with OpenCV 3. Please verify your OpenCV
|
||||
version beforehand.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/autoflip:run_autoflip
|
||||
```
|
||||
|
||||
## Running
|
||||
|
||||
```bash
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/autoflip/run_autoflip \
|
||||
--calculator_graph_config_file=mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt \
|
||||
--input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,output_video_path=/absolute/path/to/save/the/output/video/file,aspect_ratio=1:1
|
||||
```
|
||||
|
||||
Use the `aspect_ratio` flag to provide the output aspect ratio. The format
|
||||
should be `width:height`, where the `width` and `height` are two positive
|
||||
integers. AutoFlip supports both landscape-to-portrait and portrait-to-landscape
|
||||
conversions. The pipeline internally compares the target aspect ratio against
|
||||
the original one, and determines the correct conversion automatically.
|
||||
|
||||
We have put a couple test videos under this
|
||||
[Google Drive folder](https://drive.google.com/corp/drive/u/0/folders/1KK9LV--Ey0UEVpxssVLhVl7dypgJSQgk).
|
||||
You could download the videos into your local file system, then modify the
|
||||
command above accordingly to run AutoFlip against the videos.
|
||||
|
||||
## MediaPipe Graph
|
||||
|
||||
|
||||
|
||||
To visualize the graph as shown above, copy the text specification of the graph
|
||||
below and paste it into [MediaPipe Visualizer](https://viz.mediapipe.dev).
|
||||
|
||||
```bash
|
||||
# Autoflip graph that only renders the final cropped video. For use with
|
||||
# end user applications.
|
||||
max_queue_size: -1
|
||||
|
||||
# VIDEO_PREP: Decodes an input video file into images and a video header.
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_video_path"
|
||||
output_stream: "VIDEO:video_raw"
|
||||
output_stream: "VIDEO_PRESTREAM:video_header"
|
||||
output_side_packet: "SAVED_AUDIO_PATH:audio_path"
|
||||
}
|
||||
|
||||
# VIDEO_PREP: Scale the input video before feature extraction.
|
||||
node {
|
||||
calculator: "ScaleImageCalculator"
|
||||
input_stream: "FRAMES:video_raw"
|
||||
input_stream: "VIDEO_HEADER:video_header"
|
||||
output_stream: "FRAMES:video_frames_scaled"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ScaleImageCalculatorOptions]: {
|
||||
preserve_aspect_ratio: true
|
||||
output_format: SRGB
|
||||
target_width: 480
|
||||
algorithm: DEFAULT_WITHOUT_UPSCALE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# VIDEO_PREP: Create a low frame rate stream for feature extraction.
|
||||
node {
|
||||
calculator: "PacketThinnerCalculator"
|
||||
input_stream: "video_frames_scaled"
|
||||
output_stream: "video_frames_scaled_downsampled"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.PacketThinnerCalculatorOptions]: {
|
||||
thinner_type: ASYNC
|
||||
period: 200000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# DETECTION: find borders around the video and major background color.
|
||||
node {
|
||||
calculator: "BorderDetectionCalculator"
|
||||
input_stream: "VIDEO:video_raw"
|
||||
output_stream: "DETECTED_BORDERS:borders"
|
||||
}
|
||||
|
||||
# DETECTION: find shot/scene boundaries on the full frame rate stream.
|
||||
node {
|
||||
calculator: "ShotBoundaryCalculator"
|
||||
input_stream: "VIDEO:video_frames_scaled"
|
||||
output_stream: "IS_SHOT_CHANGE:shot_change"
|
||||
options {
|
||||
[type.googleapis.com/mediapipe.autoflip.ShotBoundaryCalculatorOptions] {
|
||||
min_shot_span: 0.2
|
||||
min_motion: 0.3
|
||||
window_size: 15
|
||||
min_shot_measure: 10
|
||||
min_motion_with_shot_measure: 0.05
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# DETECTION: find faces on the down sampled stream
|
||||
node {
|
||||
calculator: "AutoFlipFaceDetectionSubgraph"
|
||||
input_stream: "VIDEO:video_frames_scaled_downsampled"
|
||||
output_stream: "DETECTIONS:face_detections"
|
||||
}
|
||||
node {
|
||||
calculator: "FaceToRegionCalculator"
|
||||
input_stream: "VIDEO:video_frames_scaled_downsampled"
|
||||
input_stream: "FACES:face_detections"
|
||||
output_stream: "REGIONS:face_regions"
|
||||
}
|
||||
|
||||
# DETECTION: find objects on the down sampled stream
|
||||
node {
|
||||
calculator: "AutoFlipObjectDetectionSubgraph"
|
||||
input_stream: "VIDEO:video_frames_scaled_downsampled"
|
||||
output_stream: "DETECTIONS:object_detections"
|
||||
}
|
||||
node {
|
||||
calculator: "LocalizationToRegionCalculator"
|
||||
input_stream: "DETECTIONS:object_detections"
|
||||
output_stream: "REGIONS:object_regions"
|
||||
options {
|
||||
[type.googleapis.com/mediapipe.autoflip.LocalizationToRegionCalculatorOptions] {
|
||||
output_all_signals: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# SIGNAL FUSION: Combine detections (with weights) on each frame
|
||||
node {
|
||||
calculator: "SignalFusingCalculator"
|
||||
input_stream: "shot_change"
|
||||
input_stream: "face_regions"
|
||||
input_stream: "object_regions"
|
||||
output_stream: "salient_regions"
|
||||
options {
|
||||
[type.googleapis.com/mediapipe.autoflip.SignalFusingCalculatorOptions] {
|
||||
signal_settings {
|
||||
type { standard: FACE_CORE_LANDMARKS }
|
||||
min_score: 0.85
|
||||
max_score: 0.9
|
||||
is_required: false
|
||||
}
|
||||
signal_settings {
|
||||
type { standard: FACE_ALL_LANDMARKS }
|
||||
min_score: 0.8
|
||||
max_score: 0.85
|
||||
is_required: false
|
||||
}
|
||||
signal_settings {
|
||||
type { standard: FACE_FULL }
|
||||
min_score: 0.8
|
||||
max_score: 0.85
|
||||
is_required: false
|
||||
}
|
||||
signal_settings {
|
||||
type: { standard: HUMAN }
|
||||
min_score: 0.75
|
||||
max_score: 0.8
|
||||
is_required: false
|
||||
}
|
||||
signal_settings {
|
||||
type: { standard: PET }
|
||||
min_score: 0.7
|
||||
max_score: 0.75
|
||||
is_required: false
|
||||
}
|
||||
signal_settings {
|
||||
type: { standard: CAR }
|
||||
min_score: 0.7
|
||||
max_score: 0.75
|
||||
is_required: false
|
||||
}
|
||||
signal_settings {
|
||||
type: { standard: OBJECT }
|
||||
min_score: 0.1
|
||||
max_score: 0.2
|
||||
is_required: false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# CROPPING: make decisions about how to crop each frame.
|
||||
node {
|
||||
calculator: "SceneCroppingCalculator"
|
||||
input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
|
||||
input_stream: "VIDEO_FRAMES:video_raw"
|
||||
input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
|
||||
input_stream: "DETECTION_FEATURES:salient_regions"
|
||||
input_stream: "STATIC_FEATURES:borders"
|
||||
input_stream: "SHOT_BOUNDARIES:shot_change"
|
||||
output_stream: "CROPPED_FRAMES:cropped_frames"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.autoflip.SceneCroppingCalculatorOptions]: {
|
||||
max_scene_size: 600
|
||||
key_frame_crop_options: {
|
||||
score_aggregation_type: CONSTANT
|
||||
}
|
||||
scene_camera_motion_analyzer_options: {
|
||||
motion_stabilization_threshold_percent: 0.5
|
||||
salient_point_bound: 0.499
|
||||
}
|
||||
padding_parameters: {
|
||||
blur_cv_size: 200
|
||||
overlay_opacity: 0.6
|
||||
}
|
||||
target_size_type: MAXIMIZE_TARGET_DIMENSION
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ENCODING(required): encode the video stream for the final cropped output.
|
||||
node {
|
||||
calculator: "VideoPreStreamCalculator"
|
||||
# Fetch frame format and dimension from input frames.
|
||||
input_stream: "FRAME:cropped_frames"
|
||||
# Copying frame rate and duration from original video.
|
||||
input_stream: "VIDEO_PRESTREAM:video_header"
|
||||
output_stream: "output_frames_video_header"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "OpenCvVideoEncoderCalculator"
|
||||
input_stream: "VIDEO:cropped_frames"
|
||||
input_stream: "VIDEO_PRESTREAM:output_frames_video_header"
|
||||
input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
|
||||
input_side_packet: "AUDIO_FILE_PATH:audio_path"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
|
||||
codec: "avc1"
|
||||
video_format: "mp4"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Advanced Parameters
|
||||
|
||||
### Required vs. Best-Effort Saliency Features
|
||||
|
||||
AutoFlip allows users to implement and specify custom features to be used in the
|
||||
camera trajectory computation. If the user would like to detect and preserve
|
||||
scenes of lions in a wildlife protection video, for example, they could
|
||||
implement and add a feature detection calculator for lions into the pipeline.
|
||||
Refer to `AutoFlipFaceDetectionSubgraph` and `FaceToRegionCalculator`, or
|
||||
`AutoFlipObjectDetectionSubgraph` and `LocalizationToRegionCalculator` for
|
||||
examples of how to create new feature detection calculators.
|
||||
|
||||
After adding different feature signals into the graph, use the
|
||||
`SignalFusingCalculator` node to specify types and weights for different feature
|
||||
signals. For example, in the graph above, we specified a `face_region` and an
|
||||
`object_region` input streams, to represent face signals and agnostic object
|
||||
signals, respectively.
|
||||
|
||||
The larger the weight, the more important the features will be considered when
|
||||
AutoFlip computes the camera trajectory. Use the `is_required` flag to mark a
|
||||
feature as a hard constraint, in which case the computed camera trajectory will
|
||||
try best to cover these feature types in the cropped videos. If for some reason
|
||||
the required features cannot be all covered (for example, when they are too
|
||||
spread out in the video), AutoFlip will apply a padding effect to cover as much
|
||||
salient content as possible. See an illustration below.
|
||||
|
||||
|
||||
|
||||
### Stable vs Tracking Camera Motion
|
||||
|
||||
AutoFlip makes a decision on each scene whether to have the cropped viewpoint
|
||||
follow an object or if the crop should remain stable (centered on detected
|
||||
objects). The parameter `motion_stabilization_threshold_percent` value is used
|
||||
to make the decision to track action or keep the camera stable. If, over the
|
||||
duration of the scene, all detected focus objects remain within this ratio of
|
||||
the frame (e.g. 0.5 = 50% or 1920 * .5 = 960 pixels on 1080p video) then the
|
||||
camera is held steady. Otherwise the camera tracks activity within the frame.
|
||||
|
||||
### Snap To Center
|
||||
|
||||
For some scenes the camera viewpoint will remain stable at the center of
|
||||
activity (see `motion_stabilization_threshold_percent` setting). In this case,
|
||||
if the determined best stable viewpoint is within
|
||||
`snap_center_max_distance_percent` of the frame's center the camera will be
|
||||
shifted to be locked to the center of the frame. This setting is useful for
|
||||
videos where the camera operator did a good job already centering content or if
|
||||
titles and logos are expected to appear in the center of the frame. It may be
|
||||
less useful on raw content where objects are not already well positioned on
|
||||
screen.
|
||||
|
||||
### Visualization to Facilitate Debugging
|
||||
|
||||
`SceneCroppingCalculator` provides two extra output streams
|
||||
`KEY_FRAME_CROP_REGION_VIZ_FRAMES` and `SALIENT_POINT_FRAME_VIZ_FRAMES` to
|
||||
visualize the cropping window as well as salient points detected on each frame.
|
||||
You could modify the `SceneCroppingCalculator` node like below to enable these
|
||||
two output streams.
|
||||
|
||||
```bash
|
||||
node {
|
||||
calculator: "SceneCroppingCalculator"
|
||||
input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
|
||||
input_stream: "VIDEO_FRAMES:video_raw"
|
||||
input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
|
||||
input_stream: "DETECTION_FEATURES:salient_regions"
|
||||
input_stream: "STATIC_FEATURES:borders"
|
||||
input_stream: "SHOT_BOUNDARIES:shot_change"
|
||||
output_stream: "CROPPED_FRAMES:cropped_frames"
|
||||
output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
|
||||
output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.autoflip.SceneCroppingCalculatorOptions]: {
|
||||
max_scene_size: 600
|
||||
key_frame_crop_options: {
|
||||
score_aggregation_type: CONSTANT
|
||||
}
|
||||
scene_camera_motion_analyzer_options: {
|
||||
motion_stabilization_threshold_percent: 0.5
|
||||
salient_point_bound: 0.499
|
||||
}
|
||||
padding_parameters: {
|
||||
blur_cv_size: 200
|
||||
overlay_opacity: 0.6
|
||||
}
|
||||
target_size_type: MAXIMIZE_TARGET_DIMENSION
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,146 @@
|
||||
---
|
||||
layout: default
|
||||
title: Box Tracking
|
||||
parent: Solutions
|
||||
nav_order: 6
|
||||
---
|
||||
|
||||
# MediaPipe Box Tracking
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
MediaPipe Box Tracking has been powering real-time tracking in
|
||||
[Motion Stills](https://ai.googleblog.com/2016/12/get-moving-with-new-motion-stills.html),
|
||||
[YouTube's privacy blur](https://youtube-creators.googleblog.com/2016/02/blur-moving-objects-in-your-video-with.html),
|
||||
and [Google Lens](https://lens.google.com/) for several years, leveraging
|
||||
classic computer vision approaches.
|
||||
|
||||
The box tracking solution consumes image frames from a video or camera stream,
|
||||
and starting box positions with timestamps, indicating 2D regions of interest to
|
||||
track, and computes the tracked box positions for each frame. In this specific
|
||||
use case, the starting box positions come from object detection, but the
|
||||
starting position can also be provided manually by the user or another system.
|
||||
Our solution consists of three main components: a motion analysis component, a
|
||||
flow packager component, and a box tracking component. Each component is
|
||||
encapsulated as a MediaPipe calculator, and the box tracking solution as a whole
|
||||
is represented as a MediaPipe
|
||||
[subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/box_tracking_gpu.pbtxt).
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/).
|
||||
|
||||
In the
|
||||
[box tracking subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/box_tracking_gpu.pbtxt),
|
||||
the MotionAnalysis calculator extracts features (e.g. high-gradient corners)
|
||||
across the image, tracks those features over time, classifies them into
|
||||
foreground and background features, and estimates both local motion vectors and
|
||||
the global motion model. The FlowPackager calculator packs the estimated motion
|
||||
metadata into an efficient format. The BoxTracker calculator takes this motion
|
||||
metadata from the FlowPackager calculator and the position of starting boxes,
|
||||
and tracks the boxes over time. Using solely the motion data (without the need
|
||||
for the RGB frames) produced by the MotionAnalysis calculator, the BoxTracker
|
||||
calculator tracks individual objects or regions while discriminating from
|
||||
others. Please see
|
||||
[Object Detection and Tracking using MediaPipe](https://developers.googleblog.com/2019/12/object-detection-and-tracking-using-mediapipe.html)
|
||||
in Google Developers Blog for more details.
|
||||
|
||||
An advantage of our architecture is that by separating motion analysis into a
|
||||
dedicated MediaPipe calculator and tracking features over the whole image, we
|
||||
enable great flexibility and constant computation independent of the number of
|
||||
regions tracked! By not having to rely on the RGB frames during tracking, our
|
||||
tracking solution provides the flexibility to cache the metadata across a batch
|
||||
of frame. Caching enables tracking of regions both backwards and forwards in
|
||||
time; or even sync directly to a specified timestamp for tracking with random
|
||||
access.
|
||||
|
||||
## Object Detection and Tracking
|
||||
|
||||
MediaPipe Box Tracking can be paired with ML inference, resulting in valuable
|
||||
and efficient pipelines. For instance, box tracking can be paired with ML-based
|
||||
object detection to create an object detection and tracking pipeline. With
|
||||
tracking, this pipeline offers several advantages over running detection per
|
||||
frame (e.g., [MediaPipe Object Detection](./object_detection.md)):
|
||||
|
||||
* It provides instance based tracking, i.e. the object ID is maintained across
|
||||
frames.
|
||||
* Detection does not have to run every frame. This enables running heavier
|
||||
detection models that are more accurate while keeping the pipeline
|
||||
lightweight and real-time on mobile devices.
|
||||
* Object localization is temporally consistent with the help of tracking,
|
||||
meaning less jitter is observable across frames.
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------------: |
|
||||
*Fig 1. Box tracking paired with ML-based object detection.* |
|
||||
|
||||
The object detection and tracking pipeline can be implemented as a MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/object_detection_tracking_mobile_gpu.pbtxt),
|
||||
which internally utilizes an
|
||||
[object detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/object_detection_gpu.pbtxt),
|
||||
an
|
||||
[object tracking subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/object_tracking_gpu.pbtxt),
|
||||
and a
|
||||
[renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt).
|
||||
|
||||
In general, the object detection subgraph (which performs ML model inference
|
||||
internally) runs only upon request, e.g. at an arbitrary frame rate or triggered
|
||||
by specific signals. More specifically, in this particular
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/object_detection_tracking_mobile_gpu.pbtxt)
|
||||
a PacketResampler calculator temporally subsamples the incoming video frames to
|
||||
0.5 fps before they are passed into the object detection subgraph. This frame
|
||||
rate can be configured differently as an option in PacketResampler.
|
||||
|
||||
The object tracking subgraph runs in real-time on every incoming frame to track
|
||||
the detected objects. It expands the
|
||||
[box tracking subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/box_tracking_gpu.pbtxt)
|
||||
with additional functionality: when new detections arrive it uses IoU
|
||||
(Intersection over Union) to associate the current tracked objects/boxes with
|
||||
new detections to remove obsolete or duplicated boxes.
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android), [iOS](../getting_started/building_examples.md#ios)
|
||||
and [desktop](../getting_started/building_examples.md#desktop) on how to build MediaPipe
|
||||
examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
### Mobile
|
||||
|
||||
Note: Object detection is using TensorFlow Lite on GPU while tracking is on CPU.
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/tracking/object_detection_tracking_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/object_detection_tracking_mobile_gpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1UXL9jX4Wpp34TsiVogugV3J3T9_C5UK-)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu:objecttrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD)
|
||||
* iOS target: Not available
|
||||
|
||||
### Desktop
|
||||
|
||||
* Running on CPU (both for object detection using TensorFlow Lite and
|
||||
tracking):
|
||||
* Graph:
|
||||
[`mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/object_tracking:object_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/object_tracking/BUILD)
|
||||
* Running on GPU: Not available
|
||||
|
||||
## Resources
|
||||
|
||||
* Google Developers Blog:
|
||||
[Object Detection and Tracking using MediaPipe](https://developers.googleblog.com/2019/12/object-detection-and-tracking-using-mediapipe.html)
|
||||
* Google AI Blog:
|
||||
[Get moving with the new Motion Stills](https://ai.googleblog.com/2016/12/get-moving-with-new-motion-stills.html)
|
||||
* YouTube Creator Blog: [Blur moving objects in your video with the new Custom
|
||||
blurring tool on
|
||||
YouTube](https://youtube-creators.googleblog.com/2016/02/blur-moving-objects-in-your-video-with.html)
|
||||
@@ -0,0 +1,110 @@
|
||||
---
|
||||
layout: default
|
||||
title: Face Detection
|
||||
parent: Solutions
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
# MediaPipe Face Detection
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
MediaPipe Face Detection is an ultrafast face detection solution that comes with
|
||||
6 landmarks and multi-face support. It is based on
|
||||
[BlazeFace](https://arxiv.org/abs/1907.05047), a lightweight and well-performing
|
||||
face detector tailored for mobile GPU inference. The detector's super-realtime
|
||||
performance enables it to be applied to any live viewfinder experience that
|
||||
requires an accurate facial region of interest as an input for other
|
||||
task-specific models, such as 3D facial keypoint or geometry estimation (e.g.,
|
||||
[MediaPipe Face Mesh](./face_mesh.md)), facial features or expression
|
||||
classification, and face region segmentation. BlazeFace uses a lightweight
|
||||
feature extraction network inspired by, but distinct from
|
||||
[MobileNetV1/V2](https://ai.googleblog.com/2018/04/mobilenetv2-next-generation-of-on.html),
|
||||
a GPU-friendly anchor scheme modified from
|
||||
[Single Shot MultiBox Detector (SSD)](https://arxiv.org/abs/1512.02325), and an
|
||||
improved tie resolution strategy alternative to non-maximum suppression. For
|
||||
more information about BlazeFace, please see the [Resources](#resources)
|
||||
section.
|
||||
|
||||
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android), [iOS](../getting_started/building_examples.md#ios)
|
||||
and [desktop](../getting_started/building_examples.md#desktop) on how to build MediaPipe
|
||||
examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
### Mobile
|
||||
|
||||
#### GPU Pipeline
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1DZTCy1gp238kkMnu4fUkwI3IrF77Mhy5)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu:facedetectiongpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facedetectiongpu/BUILD)
|
||||
|
||||
#### CPU Pipeline
|
||||
|
||||
This is very similar to the [GPU pipeline](#gpu-pipeline) except that at the
|
||||
beginning and the end of the pipeline it performs GPU-to-CPU and CPU-to-GPU
|
||||
image transfer respectively. As a result, the rest of graph, which shares the
|
||||
same configuration as the GPU pipeline, runs entirely on CPU.
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1npiZY47jbO5m2YaL63o5QoCQs40JC6C7)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu:facedetectioncpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facedetectioncpu/BUILD)
|
||||
|
||||
### Desktop
|
||||
|
||||
* Running on CPU:
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/face_detection:face_detection_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/face_detection/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/face_detection:face_detection_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/face_detection/BUILD)
|
||||
|
||||
### Web
|
||||
|
||||
Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
|
||||
### Coral
|
||||
|
||||
Please refer to
|
||||
[these instructions](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
||||
to cross-compile and run MediaPipe examples on the
|
||||
[Coral Dev Board](https://coral.ai/products/dev-board).
|
||||
|
||||
## Resources
|
||||
|
||||
* Paper:
|
||||
[BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs](https://arxiv.org/abs/1907.05047)
|
||||
([presentation](https://docs.google.com/presentation/d/1YCtASfnYyZtH-41QvnW5iZxELFnf0MF-pPWSLGj8yjQ/present?slide=id.g5bc8aeffdd_1_0))
|
||||
([poster](https://drive.google.com/file/d/1u6aB6wxDY7X2TmeUUKgFydulNtXkb3pu/view))
|
||||
* For front-facing/selfie camera:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
|
||||
[TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
|
||||
* For back-facing camera:
|
||||
[TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
|
||||
* [Model card](https://drive.google.com/file/d/1f39lSzU5Oq-j_OXgS67KfN5wNsoeAZ4V/view)
|
||||
@@ -0,0 +1,160 @@
|
||||
---
|
||||
layout: default
|
||||
title: Face Mesh
|
||||
parent: Solutions
|
||||
nav_order: 2
|
||||
---
|
||||
|
||||
# MediaPipe Face Mesh
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
MediaPipe Face Mesh is a face geometry solution that estimates 468 3D face
|
||||
landmarks in real-time even on mobile devices. It employs machine learning (ML)
|
||||
to infer the 3D surface geometry, requiring only a single camera input without
|
||||
the need for a dedicated depth sensor. Utilizing lightweight model architectures
|
||||
together with GPU acceleration throughout the pipeline, the solution delivers
|
||||
real-time performance critical for live experiences. The core of the solution is
|
||||
the same as what powers
|
||||
[YouTube Stories](https://youtube-creators.googleblog.com/2018/11/introducing-more-ways-to-share-your.html)'
|
||||
creator effects, the
|
||||
[Augmented Faces API in ARCore](https://developers.google.com/ar/develop/java/augmented-faces/)
|
||||
and the
|
||||
[ML Kit Face Contour Detection API](https://firebase.google.com/docs/ml-kit/face-detection-concepts#contours).
|
||||
|
||||
 |
|
||||
:-------------------------------------------------------------: |
|
||||
*Fig 1. AR effects utilizing facial surface geometry.* |
|
||||
|
||||
## ML Pipeline
|
||||
|
||||
Our ML pipeline consists of two real-time deep neural network models that work
|
||||
together: A detector that operates on the full image and computes face locations
|
||||
and a 3D face landmark model that operates on those locations and predicts the
|
||||
approximate surface geometry via regression. Having the face accurately cropped
|
||||
drastically reduces the need for common data augmentations like affine
|
||||
transformations consisting of rotations, translation and scale changes. Instead
|
||||
it allows the network to dedicate most of its capacity towards coordinate
|
||||
prediction accuracy. In addition, in our pipeline the crops can also be
|
||||
generated based on the face landmarks identified in the previous frame, and only
|
||||
when the landmark model could no longer identify face presence is the face
|
||||
detector invoked to relocalize the face. This strategy is similar to that
|
||||
employed in our [MediaPipe Hand](./hand.md) solution, which uses a palm detector
|
||||
together with a hand landmark model.
|
||||
|
||||
The pipeline is implemented as a MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
|
||||
that uses a
|
||||
[face landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
|
||||
from the
|
||||
[face landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark),
|
||||
and renders using a dedicated
|
||||
[face renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/subgraphs/face_renderer_gpu.pbtxt).
|
||||
The
|
||||
[face landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
|
||||
internally uses a
|
||||
[face_detection_subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt)
|
||||
from the
|
||||
[face detection module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection).
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
## Models
|
||||
|
||||
### Face Detection Model
|
||||
|
||||
The face detector is the same [BlazeFace](https://arxiv.org/abs/1907.05047)
|
||||
model used in [MediaPipe Face Detection](./face_detection.md). Please refer to
|
||||
[MediaPipe Face Detection](./face_detection.md) for details.
|
||||
|
||||
### Face Landmark Model
|
||||
|
||||
For 3D face landmarks we employed transfer learning and trained a network with
|
||||
several objectives: the network simultaneously predicts 3D landmark coordinates
|
||||
on synthetic rendered data and 2D semantic contours on annotated real-world
|
||||
data. The resulting network provided us with reasonable 3D landmark predictions
|
||||
not just on synthetic but also on real-world data.
|
||||
|
||||
The 3D landmark network receives as input a cropped video frame without
|
||||
additional depth input. The model outputs the positions of the 3D points, as
|
||||
well as the probability of a face being present and reasonably aligned in the
|
||||
input. A common alternative approach is to predict a 2D heatmap for each
|
||||
landmark, but it is not amenable to depth prediction and has high computational
|
||||
costs for so many points. We further improve the accuracy and robustness of our
|
||||
model by iteratively bootstrapping and refining predictions. That way we can
|
||||
grow our dataset to increasingly challenging cases, such as grimaces, oblique
|
||||
angle and occlusions.
|
||||
|
||||
You can find more information about the face landmark model in this
|
||||
[paper](https://arxiv.org/abs/1907.06724).
|
||||
|
||||
 |
|
||||
:------------------------------------------------------------------------: |
|
||||
*Fig 2. Output of MediaPipe Face Mesh: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android), [iOS](../getting_started/building_examples.md#ios) and
|
||||
[desktop](../getting_started/building_examples.md#desktop) on how to build MediaPipe examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
### Mobile
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1pUmd7CXCL_onYMbsZo5p91cH0oNnR4gi)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu:facemeshgpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp`](http:/mediapipe/examples/ios/facemeshgpu/BUILD)
|
||||
|
||||
Tip: Maximum number of faces to detect/process is set to 1 by default. To change
|
||||
it, for Android modify `NUM_FACES` in
|
||||
[MainActivity.java](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/MainActivity.java),
|
||||
and for iOS modify `kNumFaces` in
|
||||
[ViewController.mm](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facemeshgpu/ViewController.mm).
|
||||
|
||||
### Desktop
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/face_mesh:face_mesh_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/face_mesh/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/face_mesh:face_mesh_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/face_mesh/BUILD)
|
||||
|
||||
Tip: Maximum number of faces to detect/process is set to 1 by default. To change
|
||||
it, in the graph file modify the option of `ConstantSidePacketCalculator`.
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
[Real-Time AR Self-Expression with Machine Learning](https://ai.googleblog.com/2019/03/real-time-ar-self-expression-with.html)
|
||||
* TensorFlow Blog:
|
||||
[Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
|
||||
* Paper:
|
||||
[Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724)
|
||||
([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212))
|
||||
* Face detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite)
|
||||
* Face landmark mode:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
|
||||
* [Model card](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view)
|
||||
@@ -0,0 +1,58 @@
|
||||
---
|
||||
layout: default
|
||||
title: Hair Segmentation
|
||||
parent: Solutions
|
||||
nav_order: 4
|
||||
---
|
||||
|
||||
# MediaPipe Hair Segmentation
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android), [iOS](../getting_started/building_examples.md#ios)
|
||||
and [desktop](../getting_started/building_examples.md#desktop) on how to build MediaPipe
|
||||
examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
### Mobile
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1mmLtyL8IRfCUbqqu0-E-Hgjr_e6P3XAy)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu:hairsegmentationgpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD)
|
||||
* iOS target: Not available
|
||||
|
||||
### Desktop
|
||||
|
||||
* Running on CPU: Not available
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/hair_segmentation:hair_segmentation_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/hair_segmentation/BUILD)
|
||||
|
||||
### Web
|
||||
|
||||
Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
|
||||
## Resources
|
||||
|
||||
* Paper:
|
||||
[Real-time Hair segmentation and recoloring on Mobile GPUs](https://arxiv.org/abs/1907.06740)
|
||||
([presentation](https://drive.google.com/file/d/1C8WYlWdDRNtU1_pYBvkkG5Z5wqYqf0yj/view))
|
||||
([supplementary video](https://drive.google.com/file/d/1LPtM99Ch2ogyXYbDNpEqnUfhFq0TfLuf/view))
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
|
||||
* [Model card](https://drive.google.com/file/d/1lPwJ8BD_-3UUor4LayQ0xpa_RIC_hoRh/view)
|
||||
@@ -0,0 +1,231 @@
|
||||
---
|
||||
layout: default
|
||||
title: Hand
|
||||
parent: Solutions
|
||||
nav_order: 3
|
||||
---
|
||||
|
||||
# MediaPipe Hand
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
The ability to perceive the shape and motion of hands can be a vital component
|
||||
in improving the user experience across a variety of technological domains and
|
||||
platforms. For example, it can form the basis for sign language understanding
|
||||
and hand gesture control, and can also enable the overlay of digital content and
|
||||
information on top of the physical world in augmented reality. While coming
|
||||
naturally to people, robust real-time hand perception is a decidedly challenging
|
||||
computer vision task, as hands often occlude themselves or each other (e.g.
|
||||
finger/palm occlusions and hand shakes) and lack high contrast patterns.
|
||||
|
||||
MediaPipe Hand is a high-fidelity hand and finger tracking solution. It employs
|
||||
machine learning (ML) to infer 21 3D landmarks of a hand from just a single
|
||||
frame. Whereas current state-of-the-art approaches rely primarily on powerful
|
||||
desktop environments for inference, our method achieves real-time performance on
|
||||
a mobile phone, and even scales to multiple hands. We hope that providing this
|
||||
hand perception functionality to the wider research and development community
|
||||
will result in an emergence of creative use cases, stimulating new applications
|
||||
and new research avenues.
|
||||
|
||||
 |
|
||||
:------------------------------------------------------------------------------------: |
|
||||
*Fig 1. Tracked 3D hand landmarks are represented by dots in different shades, with the brighter ones denoting landmarks closer to the camera.* |
|
||||
|
||||
## ML Pipeline
|
||||
|
||||
MediaPipe Hand utilizes an ML pipeline consisting of multiple models working
|
||||
together: A palm detection model that operates on the full image and returns an
|
||||
oriented hand bounding box. A hand landmark model that operates on the cropped
|
||||
image region defined by the palm detector and returns high-fidelity 3D hand
|
||||
keypoints. This strategy is similar to that employed in our
|
||||
[MediaPipe Face Mesh](./face_mesh.md) solution, which uses a face detector
|
||||
together with a face landmark model.
|
||||
|
||||
Providing the accurately cropped hand image to the hand landmark model
|
||||
drastically reduces the need for data augmentation (e.g. rotations, translation
|
||||
and scale) and instead allows the network to dedicate most of its capacity
|
||||
towards coordinate prediction accuracy. In addition, in our pipeline the crops
|
||||
can also be generated based on the hand landmarks identified in the previous
|
||||
frame, and only when the landmark model could no longer identify hand presence
|
||||
is palm detection invoked to relocalize the hand.
|
||||
|
||||
The pipeline is implemented as a MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt),
|
||||
which internally utilizes a
|
||||
[palm/hand detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_detection_gpu.pbtxt),
|
||||
a
|
||||
[hand landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt)
|
||||
and a
|
||||
[renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt).
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
## Models
|
||||
|
||||
### Palm Detection Model
|
||||
|
||||
To detect initial hand locations, we designed a
|
||||
[single-shot detector](https://arxiv.org/abs/1512.02325) model optimized for
|
||||
mobile real-time uses in a manner similar to the face detection model in
|
||||
[MediaPipe Face Mesh](./face_mesh.md). Detecting hands is a decidedly complex
|
||||
task: our
|
||||
[model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite) has
|
||||
to work across a variety of hand sizes with a large scale span (~20x) relative
|
||||
to the image frame and be able to detect occluded and self-occluded hands.
|
||||
Whereas faces have high contrast patterns, e.g., in the eye and mouth region,
|
||||
the lack of such features in hands makes it comparatively difficult to detect
|
||||
them reliably from their visual features alone. Instead, providing additional
|
||||
context, like arm, body, or person features, aids accurate hand localization.
|
||||
|
||||
Our method addresses the above challenges using different strategies. First, we
|
||||
train a palm detector instead of a hand detector, since estimating bounding
|
||||
boxes of rigid objects like palms and fists is significantly simpler than
|
||||
detecting hands with articulated fingers. In addition, as palms are smaller
|
||||
objects, the non-maximum suppression algorithm works well even for two-hand
|
||||
self-occlusion cases, like handshakes. Moreover, palms can be modelled using
|
||||
square bounding boxes (anchors in ML terminology) ignoring other aspect ratios,
|
||||
and therefore reducing the number of anchors by a factor of 3-5. Second, an
|
||||
encoder-decoder feature extractor is used for bigger scene context awareness
|
||||
even for small objects (similar to the RetinaNet approach). Lastly, we minimize
|
||||
the focal loss during training to support a large amount of anchors resulting
|
||||
from the high scale variance.
|
||||
|
||||
With the above techniques, we achieve an average precision of 95.7% in palm
|
||||
detection. Using a regular cross entropy loss and no decoder gives a baseline of
|
||||
just 86.22%.
|
||||
|
||||
### Hand Landmark Model
|
||||
|
||||
After the palm detection over the whole image our subsequent hand landmark
|
||||
[model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite)
|
||||
performs precise keypoint localization of 21 3D hand-knuckle coordinates inside
|
||||
the detected hand regions via regression, that is direct coordinate prediction.
|
||||
The model learns a consistent internal hand pose representation and is robust
|
||||
even to partially visible hands and self-occlusions.
|
||||
|
||||
To obtain ground truth data, we have manually annotated ~30K real-world images
|
||||
with 21 3D coordinates, as shown below (we take Z-value from image depth map, if
|
||||
it exists per corresponding coordinate). To better cover the possible hand poses
|
||||
and provide additional supervision on the nature of hand geometry, we also
|
||||
render a high-quality synthetic hand model over various backgrounds and map it
|
||||
to the corresponding 3D coordinates.
|
||||
|
||||
|  |
|
||||
| :-------------------------------------------------------------------------: |
|
||||
| *Fig 2. Top: Aligned hand crops passed to the tracking network with ground truth annotation. Bottom: Rendered synthetic hand images with ground truth annotation.* |
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android), [iOS](../getting_started/building_examples.md#ios)
|
||||
and [desktop](../getting_started/building_examples.md#desktop) on how to build MediaPipe
|
||||
examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
### Mobile
|
||||
|
||||
#### Main Example
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1uCjS0y0O0dTDItsMh8x2cf4-l3uHW1vE)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu:handtrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handtrackinggpu/BUILD)
|
||||
|
||||
#### With Multi-hand Support
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1Wk6V9EVaz1ks_MInPqqVGvvJD01SGXDc)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu:multihandtrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/multihandtrackinggpu/BUILD)
|
||||
|
||||
There are two key differences between this graph and that in the
|
||||
[main example](#main-example) (which handles only one hand):
|
||||
|
||||
1. There is a `NormalizedRectVectorHasMinSize` calculator, that checks if in
|
||||
input vector of `NormalizedRect` objects has a minimum size equal to `N`. In
|
||||
this graph, if the vector contains fewer than `N` objects,
|
||||
`MultiHandDetection` subgraph runs. Otherwise, the `GateCalculator` doesn't
|
||||
send any image packets to the `MultiHandDetection` subgraph. This way, the
|
||||
main graph is efficient in that it avoids running the costly hand detection
|
||||
step when there are already `N` hands in the frame.
|
||||
2. The `MergeCalculator` has been replaced by the `AssociationNormRect`
|
||||
calculator. This `AssociationNormRect` takes as input a vector of
|
||||
`NormalizedRect` objects from the `MultiHandDetection` subgraph on the
|
||||
current frame, and a vector of `NormalizedRect` objects from the
|
||||
`MultiHandLandmark` subgraph from the previous frame, and performs an
|
||||
association operation between these objects. This calculator ensures that
|
||||
the output vector doesn't contain overlapping regions based on the specified
|
||||
`min_similarity_threshold`.
|
||||
|
||||
#### Palm/Hand Detection Only (no landmarks)
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1qUlTtH7Ydg-wl_H6VVL8vueu2UCTu37E)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu:handdetectiongpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handdetectiongpu/BUILD)
|
||||
|
||||
### Desktop
|
||||
|
||||
#### Main Example
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/hand_tracking/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/hand_tracking/BUILD)
|
||||
|
||||
#### With Multi-hand Support
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/multi_hand_tracking/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/multi_hand_tracking/BUILD)
|
||||
|
||||
### Web
|
||||
|
||||
Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog: [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
|
||||
* TensorFlow Blog: [Face and hand tracking in the browser with MediaPipe and
|
||||
TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
|
||||
* Palm detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
|
||||
* Hand landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
|
||||
* [Model card](https://mediapipe.page.link/handmc)
|
||||
@@ -0,0 +1,145 @@
|
||||
---
|
||||
layout: default
|
||||
title: KNIFT (Template-based Feature Matching)
|
||||
parent: Solutions
|
||||
nav_order: 8
|
||||
---
|
||||
|
||||
# MediaPipe KNIFT
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
MediaPipe KNIFT is a template-based feature matching solution using KNIFT
|
||||
(Keypoint Neural Invariant Feature Transform).
|
||||
|
||||
 |
|
||||
:-----------------------------------------------------------------------: |
|
||||
*Fig 1. Matching a real Stop Sign with a Stop Sign template using KNIFT.* |
|
||||
|
||||
In many computer vision applications, a crucial building block is to establish
|
||||
reliable correspondences between different views of an object or scene, forming
|
||||
the foundation for approaches like template matching, image retrieval and
|
||||
structure from motion. Correspondences are usually computed by extracting
|
||||
distinctive view-invariant features such as
|
||||
[SIFT](https://en.wikipedia.org/wiki/Scale-invariant_feature_transform) or
|
||||
[ORB](https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_feature2d/py_orb/py_orb.html#orb-in-opencv)
|
||||
from images. The ability to reliably establish such correspondences enables
|
||||
applications like image stitching to create panoramas or template matching for
|
||||
object recognition in videos.
|
||||
|
||||
KNIFT is a general purpose local feature descriptor similar to SIFT or ORB.
|
||||
Likewise, KNIFT is also a compact vector representation of local image patches
|
||||
that is invariant to uniform scaling, orientation, and illumination changes.
|
||||
However unlike SIFT or ORB, which were engineered with heuristics, KNIFT is an
|
||||
[embedding](https://developers.google.com/machine-learning/crash-course/embeddings/video-lecture)
|
||||
learned directly from a large number of corresponding local patches extracted
|
||||
from nearby video frames. This data driven approach implicitly encodes complex,
|
||||
real-world spatial transformations and lighting changes in the embedding. As a
|
||||
result, the KNIFT feature descriptor appears to be more robust, not only to
|
||||
[affine distortions](https://en.wikipedia.org/wiki/Affine_transformation), but
|
||||
to some degree of
|
||||
[perspective distortions](https://en.wikipedia.org/wiki/Perspective_distortion_\(photography\))
|
||||
as well.
|
||||
|
||||
For more information, please see
|
||||
[MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
|
||||
in Google Developers Blog.
|
||||
|
||||
 |
|
||||
:-------------------------------------------------------------------------------------: |
|
||||
*Fig 2. Matching US dollar bills using KNIFT.* |
|
||||
|
||||
## Example Apps
|
||||
|
||||
### Matching US Dollar Bills
|
||||
|
||||
In MediaPipe, we've already provided an
|
||||
[index file](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_index.pb)
|
||||
pre-computed from the 3 template images (of US dollar bills) shown below. If
|
||||
you'd like to use your own template images, see
|
||||
[Matching Your Own Template Images](#matching-your-own-template-images).
|
||||
|
||||
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android) on how to build MediaPipe examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/template_matching/template_matching_mobile_cpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/template_matching/template_matching_mobile_cpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1tSWRfes9rAM4NrzmJBplguNQQvaeBZSa)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu:templatematchingcpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/BUILD)
|
||||
|
||||
Note: MediaPipe uses OpenCV 3 by default. However, because of
|
||||
[issues](https://github.com/opencv/opencv/issues/11488) between NDK 17+ and
|
||||
OpenCV 3 when using
|
||||
[knnMatch](https://docs.opencv.org/3.4/db/d39/classcv_1_1DescriptorMatcher.html#a378f35c9b1a5dfa4022839a45cdf0e89),
|
||||
for this example app please use the following commands to temporarily switch to
|
||||
OpenCV 4, and switch back to OpenCV 3 afterwards.
|
||||
|
||||
```bash
|
||||
# Switch to OpenCV 4
|
||||
sed -i -e 's:3.4.3/opencv-3.4.3:4.0.1/opencv-4.0.1:g' WORKSPACE
|
||||
sed -i -e 's:libopencv_java3:libopencv_java4:g' third_party/opencv_android.BUILD
|
||||
|
||||
# Build and install app
|
||||
bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu
|
||||
adb install -r bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/templatematchingcpu.apk
|
||||
|
||||
# Switch back to OpenCV 3
|
||||
sed -i -e 's:4.0.1/opencv-4.0.1:3.4.3/opencv-3.4.3:g' WORKSPACE
|
||||
sed -i -e 's:libopencv_java4:libopencv_java3:g' third_party/opencv_android.BUILD
|
||||
```
|
||||
|
||||
Tip: The example uses the TFLite
|
||||
[XNNPACK delegate](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/delegates/xnnpack)
|
||||
by default for faster inference. Users can change the
|
||||
[option in TfLiteInferenceCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/calculators/tflite/tflite_inference_calculator.proto)
|
||||
to run regular TFLite inference.
|
||||
|
||||
### Matching Your Own Template Images
|
||||
|
||||
* Step 1: Put all template images in a single directory.
|
||||
|
||||
* Step 2: To build the index file for all templates in the directory, run
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
|
||||
mediapipe/examples/desktop/template_matching:template_matching_tflite
|
||||
```
|
||||
|
||||
```bash
|
||||
bazel-bin/mediapipe/examples/desktop/template_matching/template_matching_tflite \
|
||||
--calculator_graph_config_file=mediapipe/graphs/template_matching/index_building.pbtxt \
|
||||
--input_side_packets="file_directory=<template image directory>,file_suffix=png,output_index_filename=<output index filename>"
|
||||
```
|
||||
|
||||
The output index file includes the extracted KNIFT features.
|
||||
|
||||
* Step 3: Replace
|
||||
[mediapipe/models/knift_index.pb](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_index.pb)
|
||||
with the index file you generated, and update
|
||||
[mediapipe/models/knift_labelmap.txt](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_labelmap.txt)
|
||||
with your own template names.
|
||||
|
||||
* Step 4: Build and run the app using the same instructions in
|
||||
[Matching US Dollar Bills](#matching-us-dollar-bills).
|
||||
|
||||
## Resources
|
||||
|
||||
* Google Developers Blog:
|
||||
[MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
|
||||
* [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
|
||||
* [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
|
||||
* [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
|
||||
* [Model card](https://mediapipe.page.link/knift-mc)
|
||||
@@ -0,0 +1,204 @@
|
||||
---
|
||||
layout: default
|
||||
title: Dataset Preparation with MediaSequence
|
||||
parent: Solutions
|
||||
nav_order: 10
|
||||
---
|
||||
|
||||
# Dataset Preparation with MediaSequence
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
MediaPipe is a useful and general framework for media processing that can
|
||||
assist with research, development, and deployment of ML models. This example
|
||||
focuses on development by demonstrating how to prepare video data for training
|
||||
a TensorFlow model.
|
||||
|
||||
The MediaSequence library provides an extensive set of tools for storing data in
|
||||
TensorFlow.SequenceExamples. SequenceExamples provide matched semantics to most
|
||||
video tasks and are efficient to use with TensorFlow. The sequence semantics
|
||||
allow for a variable number of annotations per frame, which is necessary for
|
||||
tasks like video object detection, but very difficult to encode in
|
||||
TensorFlow.Examples. The goal of MediaSequence is to simplify working with
|
||||
SequenceExamples and to automate common preparation tasks. Much more information
|
||||
is available about the MediaSequence pipeline, including how to use it to
|
||||
process new data sets, in the documentation of
|
||||
[MediaSequence](https://github.com/google/mediapipe/tree/master/mediapipe/util/sequence).
|
||||
|
||||
## Preparing an example data set
|
||||
|
||||
1. Checkout the mediapipe repository
|
||||
|
||||
```bash
|
||||
git clone https://github.com/google/mediapipe.git
|
||||
cd mediapipe
|
||||
```
|
||||
|
||||
1. Compile the MediaSequence demo C++ binary
|
||||
|
||||
```bash
|
||||
bazel build -c opt mediapipe/examples/desktop/media_sequence:media_sequence_demo --define MEDIAPIPE_DISABLE_GPU=1
|
||||
```
|
||||
|
||||
MediaSequence uses C++ binaries to improve multimedia processing speed and
|
||||
encourage a strong separation between annotations and the image data or
|
||||
other features. The binary code is very general in that it reads from files
|
||||
into input side packets and writes output side packets to files when
|
||||
completed, but it also links in all of the calculators for necessary for the
|
||||
MediaPipe graphs preparing the Charades data set.
|
||||
|
||||
1. Download and prepare the data set through Python
|
||||
|
||||
To run this step, you must have Python 2.7 or 3.5+ installed with the
|
||||
TensorFlow 1.14+ package installed.
|
||||
|
||||
```bash
|
||||
python -m mediapipe.examples.desktop.media_sequence.demo_dataset \
|
||||
--path_to_demo_data=/tmp/demo_data/ \
|
||||
--path_to_mediapipe_binary=bazel-bin/mediapipe/examples/desktop/media_sequence/media_sequence_demo \
|
||||
--path_to_graph_directory=mediapipe/graphs/media_sequence/
|
||||
```
|
||||
|
||||
The arguments define where data is stored. `--path_to_demo_data` defines
|
||||
where the data will be downloaded to and where prepared data will be
|
||||
generated. `--path_to_mediapipe_binary` is the path to the binary built in
|
||||
the previous step. `--path_to_graph_directory` defines where to look for
|
||||
MediaPipe graphs during processing.
|
||||
|
||||
Running this module
|
||||
|
||||
1. Downloads videos from the internet.
|
||||
1. For each annotation in a CSV, creates a structured metadata file.
|
||||
1. Runs MediaPipe to extract images as defined by the metadata.
|
||||
1. Stores the results in numbered set of TFRecords files.
|
||||
|
||||
MediaSequence uses SequenceExamples as the format of both inputs and
|
||||
outputs. Annotations are encoded as inputs in a SequenceExample of metadata
|
||||
that defines the labels and the path to the cooresponding video file. This
|
||||
metadata is passed as input to the C++ `media_sequence_demo` binary, and the
|
||||
output is a SequenceExample filled with images and annotations ready for
|
||||
model training.
|
||||
|
||||
1. Reading the data in TensorFlow
|
||||
|
||||
To read the data in tensorflow, first add the repo to your PYTHONPATH
|
||||
|
||||
```bash
|
||||
PYTHONPATH="${PYTHONPATH};"+`pwd`
|
||||
```
|
||||
|
||||
and then you can import the data set in Python using
|
||||
[read_demo_dataset.py](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/media_sequence/read_demo_dataset.py)
|
||||
|
||||
## Preparing a practical data set
|
||||
|
||||
As an example of processing a practical data set, a similar set of commands will
|
||||
prepare the [Charades data set](https://allenai.org/plato/charades/). The
|
||||
Charades data set is a data set of human action recognition collected with and
|
||||
maintained by the Allen Institute for Artificial Intelligence. To follow this
|
||||
code lab, you must abide by the
|
||||
[license](https://allenai.org/plato/charades/license.txt) for the Charades data
|
||||
set provided by the Allen Institute.
|
||||
|
||||
The Charades data set is large (~150 GB), and will take considerable time to
|
||||
download and process (4-8 hours).
|
||||
|
||||
```bash
|
||||
bazel build -c opt mediapipe/examples/desktop/media_sequence:media_sequence_demo --define MEDIAPIPE_DISABLE_GPU=1
|
||||
|
||||
python -m mediapipe.examples.desktop.media_sequence.charades_dataset \
|
||||
--alsologtostderr \
|
||||
--path_to_charades_data=/tmp/demo_data/ \
|
||||
--path_to_mediapipe_binary=bazel-bin/mediapipe/examples/desktop/media_sequence/media_sequence_demo \
|
||||
--path_to_graph_directory=mediapipe/graphs/media_sequence/
|
||||
```
|
||||
|
||||
## Preparing your own data set
|
||||
|
||||
The process for preparing your own data set is described in the
|
||||
[MediaSequence documentation](https://github.com/google/mediapipe/tree/master/mediapipe/util/sequence).
|
||||
The Python code for Charades can easily be modified to process most annotations,
|
||||
but the MediaPipe processing warrants further discussion. MediaSequence uses
|
||||
MediaPipe graphs to extract features related to the metadata or previously
|
||||
extracted data. Each graph can focus on extracting a single type of feature, and
|
||||
graphs can be chained together to extract derived features in a composable way.
|
||||
For example, one graph may extract images from a video at 10 fps and another
|
||||
graph extract images at 24 fps. A subsequent graph can extract ResNet-50
|
||||
features from the output of either preceding graph. MediaPipe enables a
|
||||
composable interface of data process for machine learning at multiple levels.
|
||||
|
||||
The MediaPipe graph with brief annotations for adding images to a data set is as
|
||||
follows. Common changes would be to change the frame_rate or encoding quality of
|
||||
frames.
|
||||
|
||||
```
|
||||
# Convert the string input into a decoded SequenceExample.
|
||||
node {
|
||||
calculator: "StringToSequenceExampleCalculator"
|
||||
input_side_packet: "STRING:input_sequence_example"
|
||||
output_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
|
||||
}
|
||||
|
||||
# Unpack the data path and clip timing from the SequenceExample.
|
||||
node {
|
||||
calculator: "UnpackMediaSequenceCalculator"
|
||||
input_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
|
||||
output_side_packet: "DATA_PATH:input_video_path"
|
||||
output_side_packet: "RESAMPLER_OPTIONS:packet_resampler_options"
|
||||
options {
|
||||
[type.googleapis.com/mediapipe.UnpackMediaSequenceCalculatorOptions]: {
|
||||
base_packet_resampler_options {
|
||||
frame_rate: 24.0
|
||||
base_timestamp: 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decode the entire video.
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_video_path"
|
||||
output_stream: "VIDEO:decoded_frames"
|
||||
}
|
||||
|
||||
# Extract the subset of frames we want to keep.
|
||||
node {
|
||||
calculator: "PacketResamplerCalculator"
|
||||
input_stream: "decoded_frames"
|
||||
output_stream: "sampled_frames"
|
||||
input_side_packet: "OPTIONS:packet_resampler_options"
|
||||
}
|
||||
|
||||
# Encode the images to store in the SequenceExample.
|
||||
node {
|
||||
calculator: "OpenCvImageEncoderCalculator"
|
||||
input_stream: "sampled_frames"
|
||||
output_stream: "encoded_frames"
|
||||
node_options {
|
||||
[type.googleapis.com/mediapipe.OpenCvImageEncoderCalculatorOptions]: {
|
||||
quality: 80
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Store the images in the SequenceExample.
|
||||
node {
|
||||
calculator: "PackMediaSequenceCalculator"
|
||||
input_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
|
||||
output_side_packet: "SEQUENCE_EXAMPLE:sequence_example_to_serialize"
|
||||
input_stream: "IMAGE:encoded_frames"
|
||||
}
|
||||
|
||||
# Serialize the SequenceExample to a string for storage.
|
||||
node {
|
||||
calculator: "StringToSequenceExampleCalculator"
|
||||
input_side_packet: "SEQUENCE_EXAMPLE:sequence_example_to_serialize"
|
||||
output_side_packet: "STRING:output_sequence_example"
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,150 @@
|
||||
---
|
||||
layout: default
|
||||
title: Object Detection
|
||||
parent: Solutions
|
||||
nav_order: 5
|
||||
---
|
||||
|
||||
# MediaPipe Object Detection
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
|
||||
|
||||
## Example Apps
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
### Mobile
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android) and
|
||||
[iOS](../getting_started/building_examples.md#ios) on how to build MediaPipe examples.
|
||||
|
||||
#### GPU Pipeline
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1di2ywCA_acf3y5rIcJHngWHAUNsUHAGz)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu:objectdetectiongpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/objectdetectiongpu/BUILD)
|
||||
|
||||
#### CPU Pipeline
|
||||
|
||||
This is very similar to the [GPU pipeline](#gpu-pipeline) except that at the
|
||||
beginning and the end of the pipeline it performs GPU-to-CPU and CPU-to-GPU
|
||||
image transfer respectively. As a result, the rest of graph, which shares the
|
||||
same configuration as the GPU pipeline, runs entirely on CPU.
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt))
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1eRBK6V5Qd1LCRwexitR2OXgrBBXbOfZ5)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu:objectdetectioncpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/objectdetectioncpu/BUILD)
|
||||
|
||||
### Desktop
|
||||
|
||||
#### Live Camera Input
|
||||
|
||||
Please first see general instructions for
|
||||
[desktop](../getting_started/building_examples.md#desktop) on how to build MediaPipe examples.
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/object_detection:object_detection_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/object_detection/BUILD)
|
||||
|
||||
#### Video File Input
|
||||
|
||||
* With a TFLite Model
|
||||
|
||||
This uses the same
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
|
||||
(see also
|
||||
[model info](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md))
|
||||
as in [Live Camera Input](#live-camera-input) above. The pipeline is
|
||||
implemented in this
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt),
|
||||
which differs from the live-camera-input CPU-based pipeline
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt)
|
||||
simply by the additional `OpenCvVideoDecoderCalculator` and
|
||||
`OpenCvVideoEncoderCalculator` at the beginning and the end of the graph
|
||||
respectively.
|
||||
|
||||
To build the application, run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/object_detection:object_detection_tflite
|
||||
```
|
||||
|
||||
To run the application, replace `<input video path>` and `<output video
|
||||
path>` in the command below with your own paths:
|
||||
|
||||
Tip: You can find a test video available in
|
||||
`mediapipe/examples/desktop/object_detection`.
|
||||
|
||||
```
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt \
|
||||
--input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>
|
||||
```
|
||||
|
||||
* With a TensorFlow Model
|
||||
|
||||
This uses the
|
||||
[TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
|
||||
( see also
|
||||
[model info](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)),
|
||||
and the pipeline is implemented in this
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt).
|
||||
|
||||
Note: The following runs TensorFlow inference on CPU. If you would like to
|
||||
run inference on GPU (Linux only), please follow
|
||||
[TensorFlow CUDA Support and Setup on Linux Desktop](gpu.md#tensorflow-cuda-support-and-setup-on-linux-desktop)
|
||||
instead.
|
||||
|
||||
To build the TensorFlow CPU inference example on desktop, run:
|
||||
|
||||
Note: This command also builds TensorFlow targets from scratch, and it may
|
||||
take a long time (e.g., up to 30 mins) for the first time.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true --linkopt=-s \
|
||||
mediapipe/examples/desktop/object_detection:object_detection_tensorflow
|
||||
```
|
||||
|
||||
To run the application, replace `<input video path>` and `<output video
|
||||
path>` in the command below with your own paths:
|
||||
|
||||
Tip: You can find a test video available in
|
||||
`mediapipe/examples/desktop/object_detection`.
|
||||
|
||||
```bash
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt \
|
||||
--input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>
|
||||
```
|
||||
|
||||
### Coral
|
||||
|
||||
Please refer to
|
||||
[these instructions](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
||||
to cross-compile and run MediaPipe examples on the
|
||||
[Coral Dev Board](https://coral.ai/products/dev-board).
|
||||
|
||||
## Resources
|
||||
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
|
||||
* [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
|
||||
* [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
|
||||
* [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
|
||||
@@ -0,0 +1,192 @@
|
||||
---
|
||||
layout: default
|
||||
title: Objectron (3D Object Detection)
|
||||
parent: Solutions
|
||||
nav_order: 7
|
||||
---
|
||||
|
||||
# MediaPipe Objectron
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
MediaPipe Objectron is a mobile real-time 3D object detection solution for
|
||||
everyday objects. It detects objects in 2D images, and estimates their poses and
|
||||
sizes through a machine learning (ML) model, trained on a newly created 3D
|
||||
dataset.
|
||||
|
||||
 | 
|
||||
:--------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------:
|
||||
*Fig 1(a). Objectron for Shoes.* | *Fig 1(b). Objectron for Chairs.*
|
||||
|
||||
Object detection is an extensively studied computer vision problem, but most of
|
||||
the research has focused on
|
||||
[2D object prediction](https://ai.googleblog.com/2017/06/supercharge-your-computer-vision-models.html).
|
||||
While 2D prediction only provides 2D bounding boxes, by extending prediction to
|
||||
3D, one can capture an object’s size, position and orientation in the world,
|
||||
leading to a variety of applications in robotics, self-driving vehicles, image
|
||||
retrieval, and augmented reality. Although 2D object detection is relatively
|
||||
mature and has been widely used in the industry, 3D object detection from 2D
|
||||
imagery is a challenging problem, due to the lack of data and diversity of
|
||||
appearances and shapes of objects within a category.
|
||||
|
||||
 |
|
||||
:-----------------------------------------------------------------------: |
|
||||
*Fig 2. Objectron example results.* |
|
||||
|
||||
## Obtaining Real-World 3D Training Data
|
||||
|
||||
While there are ample amounts of 3D data for street scenes, due to the
|
||||
popularity of research into self-driving cars that rely on 3D capture sensors
|
||||
like LIDAR, datasets with ground truth 3D annotations for more granular everyday
|
||||
objects are extremely limited. To overcome this problem, we developed a novel
|
||||
data pipeline using mobile augmented reality (AR) session data. With the arrival
|
||||
of [ARCore](https://developers.google.com/ar) and
|
||||
[ARKit](https://developer.apple.com/augmented-reality/),
|
||||
[hundreds of millions](https://arinsider.co/2019/05/13/arcore-reaches-400-million-devices/)
|
||||
of smartphones now have AR capabilities and the ability to capture additional
|
||||
information during an AR session, including the camera pose, sparse 3D point
|
||||
clouds, estimated lighting, and planar surfaces.
|
||||
|
||||
In order to label ground truth data, we built a novel annotation tool for use
|
||||
with AR session data, which allows annotators to quickly label 3D bounding boxes
|
||||
for objects. This tool uses a split-screen view to display 2D video frames on
|
||||
which are overlaid 3D bounding boxes on the left, alongside a view showing 3D
|
||||
point clouds, camera positions and detected planes on the right. Annotators draw
|
||||
3D bounding boxes in the 3D view, and verify its location by reviewing the
|
||||
projections in 2D video frames. For static objects, we only need to annotate an
|
||||
object in a single frame and propagate its location to all frames using the
|
||||
ground truth camera pose information from the AR session data, which makes the
|
||||
procedure highly efficient.
|
||||
|
||||
|  |
|
||||
| :--------------------------------------------------------------------------: |
|
||||
| *Fig 3. Real-world data annotation for 3D object detection. (Right) 3D bounding boxes are annotated in the 3D world with detected surfaces and point clouds. (Left) Projections of annotated 3D bounding boxes are overlaid on top of video frames making it easy to validate the annotation.* |
|
||||
|
||||
## AR Synthetic Data Generation
|
||||
|
||||
A popular approach is to complement real-world data with synthetic data in order
|
||||
to increase the accuracy of prediction. However, attempts to do so often yield
|
||||
poor, unrealistic data or, in the case of photorealistic rendering, require
|
||||
significant effort and compute. Our novel approach, called AR Synthetic Data
|
||||
Generation, places virtual objects into scenes that have AR session data, which
|
||||
allows us to leverage camera poses, detected planar surfaces, and estimated
|
||||
lighting to generate placements that are physically probable and with lighting
|
||||
that matches the scene. This approach results in high-quality synthetic data
|
||||
with rendered objects that respect the scene geometry and fit seamlessly into
|
||||
real backgrounds. By combining real-world data and AR synthetic data, we are
|
||||
able to increase the accuracy by about 10%.
|
||||
|
||||
 |
|
||||
:-------------------------------------------------------------------------------------------: |
|
||||
*Fig 4. An example of AR synthetic data generation. The virtual white-brown cereal box is rendered into the real scene, next to the real blue book.* |
|
||||
|
||||
## ML Model for 3D Object Detection
|
||||
|
||||
 |
|
||||
:---------------------------------------------------------------------------------: |
|
||||
*Fig 5. Network architecture and post-processing for 3D object detection.* |
|
||||
|
||||
We [built a single-stage model](https://arxiv.org/abs/2003.03522) to predict the
|
||||
pose and physical size of an object from a single RGB image. The model backbone
|
||||
has an encoder-decoder architecture, built upon
|
||||
[MobileNetv2](https://ai.googleblog.com/2018/04/mobilenetv2-next-generation-of-on.html).
|
||||
We employ a multi-task learning approach, jointly predicting an object's shape
|
||||
with detection and regression. The shape task predicts the object's shape
|
||||
signals depending on what ground truth annotation is available, e.g.
|
||||
segmentation. This is optional if there is no shape annotation in training data.
|
||||
For the detection task, we use the annotated bounding boxes and fit a Gaussian
|
||||
to the box, with center at the box centroid, and standard deviations
|
||||
proportional to the box size. The goal for detection is then to predict this
|
||||
distribution with its peak representing the object’s center location. The
|
||||
regression task estimates the 2D projections of the eight bounding box vertices.
|
||||
To obtain the final 3D coordinates for the bounding box, we leverage a well
|
||||
established pose estimation algorithm
|
||||
([EPnP](https://www.epfl.ch/labs/cvlab/software/multi-view-stereo/epnp/)). It
|
||||
can recover the 3D bounding box of an object, without a priori knowledge of the
|
||||
object dimensions. Given the 3D bounding box, we can easily compute pose and
|
||||
size of the object. The model is light enough to run real-time on mobile devices
|
||||
(at 26 FPS on an Adreno 650 mobile GPU).
|
||||
|
||||
 |
|
||||
:-------------------------------------------------------------------------------------: |
|
||||
*Fig 6. Sample results of our network — (Left) original 2D image with estimated bounding boxes, (Middle) object detection by Gaussian distribution, (Right) predicted segmentation mask.* |
|
||||
|
||||
## Detection and Tracking Pipeline
|
||||
|
||||
When the model is applied to every frame captured by the mobile device, it can
|
||||
suffer from jitter due to the ambiguity of the 3D bounding box estimated in each
|
||||
frame. To mitigate this, we adopt the same detection+tracking strategy in our
|
||||
[2D object detection and tracking pipeline](./box_tracking.md#object-detection-and-tracking)
|
||||
in [MediaPipe Box Tracking](./box_tracking.md). This mitigates the need to run
|
||||
the network on every frame, allowing the use of heavier and therefore more
|
||||
accurate models, while keeping the pipeline real-time on mobile devices. It also
|
||||
retains object identity across frames and ensures that the prediction is
|
||||
temporally consistent, reducing the jitter.
|
||||
|
||||
The Objectron 3D object detection and tracking pipeline is implemented as a
|
||||
MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt),
|
||||
which internally uses a
|
||||
[detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/subgraphs/objectron_detection_gpu.pbtxt)
|
||||
and a
|
||||
[tracking subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/subgraphs/objectron_tracking_gpu.pbtxt).
|
||||
The detection subgraph performs ML inference only once every few frames to
|
||||
reduce computation load, and decodes the output tensor to a FrameAnnotation that
|
||||
contains nine keypoints: the 3D bounding box's center and its eight vertices.
|
||||
The tracking subgraph runs every frame, using the box traker in
|
||||
[MediaPipe Box Tracking](./box_tracking.md) to track the 2D box tightly
|
||||
enclosing the projection of the 3D bounding box, and lifts the tracked 2D
|
||||
keypoints to 3D with
|
||||
[EPnP](https://www.epfl.ch/labs/cvlab/software/multi-view-stereo/epnp/). When
|
||||
new detection becomes available from the detection subgraph, the tracking
|
||||
subgraph is also responsible for consolidation between the detection and
|
||||
tracking results, based on the area of overlap.
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android) and
|
||||
[iOS](../getting_started/building_examples.md#ios) on how to build MediaPipe examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../visualizer.md).
|
||||
|
||||
### Objectron for Shoes
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1S0K4hbWt3o31FfQ4QU3Rz7IHrvOUMx1d)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD)
|
||||
* iOS target: Not available
|
||||
|
||||
### Objectron for Chairs
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/chair_classic_occlusion_tracking.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1MM8K-13bXLCVS1EHQ-KgkVyEahEPrKej)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD)
|
||||
and add `--define chair=true` to the build command, i.e.,
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define chair=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
[Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
|
||||
* Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
|
||||
Shape Supervision](https://arxiv.org/abs/2003.03522)
|
||||
* [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
|
||||
* [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
|
||||
@@ -0,0 +1,31 @@
|
||||
---
|
||||
layout: default
|
||||
title: Solutions
|
||||
nav_order: 3
|
||||
has_children: true
|
||||
has_toc: false
|
||||
---
|
||||
|
||||
# Solutions
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to ../index.md. -->
|
||||
|
||||
[]() | Android | iOS | Desktop | Web | Coral
|
||||
:---------------------------------------------------------------------------- | :-----: | :-: | :-----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | |
|
||||
[Hand](https://google.github.io/mediapipe/solutions/hand) | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | |
|
||||
@@ -0,0 +1,189 @@
|
||||
---
|
||||
layout: default
|
||||
title: YouTube-8M Feature Extraction and Model Inference
|
||||
parent: Solutions
|
||||
nav_order: 11
|
||||
---
|
||||
|
||||
# YouTube-8M Feature Extraction and Model Inference
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
MediaPipe is a useful and general framework for media processing that can assist
|
||||
with research, development, and deployment of ML models. This example focuses on
|
||||
model development by demonstrating how to prepare training data and do model
|
||||
inference for the YouTube-8M Challenge.
|
||||
|
||||
## Extracting Video Features for YouTube-8M Challenge
|
||||
|
||||
[Youtube-8M Challenge](https://www.kaggle.com/c/youtube8m-2019) is an annual
|
||||
video classification challenge hosted by Google. Over the last two years, the
|
||||
first two challenges have collectively drawn 1000+ teams from 60+ countries to
|
||||
further advance large-scale video understanding research. In addition to the
|
||||
feature extraction Python code released in the
|
||||
[google/youtube-8m](https://github.com/google/youtube-8m/tree/master/feature_extractor)
|
||||
repo, we release a MediaPipe based feature extraction pipeline that can extract
|
||||
both video and audio features from a local video. The MediaPipe based pipeline
|
||||
utilizes two machine learning models,
|
||||
[Inception v3](https://github.com/tensorflow/models/tree/master/research/inception)
|
||||
and
|
||||
[VGGish](https://github.com/tensorflow/models/tree/master/research/audioset/vggish),
|
||||
to extract features from video and audio respectively.
|
||||
|
||||
To visualize the
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/youtube8m/feature_extraction.pbtxt),
|
||||
copy the text specification of the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). The feature extraction
|
||||
pipeline is highly customizable. You are welcome to add new calculators or use
|
||||
your own machine learning models to extract more advanced features from the
|
||||
videos.
|
||||
|
||||
### Steps to run the YouTube-8M feature extraction graph
|
||||
|
||||
1. Checkout the repository and follow
|
||||
[the installation instructions](https://github.com/google/mediapipe/blob/master/mediapipe/docs/install.md)
|
||||
to set up MediaPipe.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/google/mediapipe.git
|
||||
cd mediapipe
|
||||
```
|
||||
|
||||
2. Download the PCA and model data.
|
||||
|
||||
```bash
|
||||
mkdir /tmp/mediapipe
|
||||
cd /tmp/mediapipe
|
||||
curl -O http://data.yt8m.org/pca_matrix_data/inception3_mean_matrix_data.pb
|
||||
curl -O http://data.yt8m.org/pca_matrix_data/inception3_projection_matrix_data.pb
|
||||
curl -O http://data.yt8m.org/pca_matrix_data/vggish_mean_matrix_data.pb
|
||||
curl -O http://data.yt8m.org/pca_matrix_data/vggish_projection_matrix_data.pb
|
||||
curl -O http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz
|
||||
tar -xvf /tmp/mediapipe/inception-2015-12-05.tgz
|
||||
```
|
||||
|
||||
3. Get the VGGish frozen graph.
|
||||
|
||||
Note: To run step 3 and step 4, you must have Python 2.7 or 3.5+ installed
|
||||
with the TensorFlow 1.14+ package installed.
|
||||
|
||||
```bash
|
||||
# cd to the root directory of the MediaPipe repo
|
||||
cd -
|
||||
|
||||
pip3 install tf_slim
|
||||
python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph
|
||||
```
|
||||
|
||||
4. Generate a MediaSequence metadata from the input video.
|
||||
|
||||
Note: the output file is /tmp/mediapipe/metadata.pb
|
||||
|
||||
```bash
|
||||
# change clip_end_time_sec to match the length of your video.
|
||||
python -m mediapipe.examples.desktop.youtube8m.generate_input_sequence_example \
|
||||
--path_to_input_video=/absolute/path/to/the/local/video/file \
|
||||
--clip_end_time_sec=120
|
||||
```
|
||||
|
||||
5. Run the MediaPipe binary to extract the features.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --linkopt=-s \
|
||||
--define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \
|
||||
mediapipe/examples/desktop/youtube8m:extract_yt8m_features
|
||||
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/extract_yt8m_features \
|
||||
--calculator_graph_config_file=mediapipe/graphs/youtube8m/feature_extraction.pbtxt \
|
||||
--input_side_packets=input_sequence_example=/tmp/mediapipe/metadata.pb \
|
||||
--output_side_packets=output_sequence_example=/tmp/mediapipe/features.pb
|
||||
```
|
||||
|
||||
6. [Optional] Read the features.pb in Python.
|
||||
|
||||
```
|
||||
import tensorflow as tf
|
||||
|
||||
sequence_example = open('/tmp/mediapipe/features.pb', 'rb').read()
|
||||
print(tf.train.SequenceExample.FromString(sequence_example))
|
||||
```
|
||||
|
||||
## Model Inference for YouTube-8M Challenge
|
||||
|
||||
MediaPipe can help you do model inference for YouTube-8M Challenge with both
|
||||
local videos and the YouTube-8M dataset. To visualize
|
||||
[the graph for local videos](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt)
|
||||
and
|
||||
[the graph for the YouTube-8M dataset](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/youtube8m/yt8m_dataset_model_inference.pbtxt),
|
||||
copy the text specification of the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). We use the baseline model
|
||||
[(model card)](https://drive.google.com/file/d/1xTCi9-Nm9dt2KIk8WR0dDFrIssWawyXy/view)
|
||||
in our example. But, the model inference pipeline is highly customizable. You
|
||||
are welcome to add new calculators or use your own machine learning models to do
|
||||
the inference for both local videos and the dataset
|
||||
|
||||
### Steps to run the YouTube-8M model inference graph with Web Interface
|
||||
|
||||
1. Copy the baseline model
|
||||
[(model card)](https://drive.google.com/file/d/1xTCi9-Nm9dt2KIk8WR0dDFrIssWawyXy/view)
|
||||
to local.
|
||||
|
||||
```bash
|
||||
curl -o /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz http://data.yt8m.org/models/baseline/saved_model.tar.gz
|
||||
|
||||
tar -xvf /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz -C /tmp/mediapipe
|
||||
```
|
||||
|
||||
2. Build the inference binary.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
|
||||
mediapipe/examples/desktop/youtube8m:model_inference
|
||||
```
|
||||
|
||||
3. Run the python web server.
|
||||
|
||||
Note: pip3 install absl-py
|
||||
|
||||
```bash
|
||||
python mediapipe/examples/desktop/youtube8m/viewer/server.py --root `pwd`
|
||||
```
|
||||
|
||||
Navigate to localhost:8008 in a web browser.
|
||||
[Here](https://drive.google.com/file/d/19GSvdAAuAlACpBhHOaqMWZ_9p8bLUYKh/view?usp=sharing)
|
||||
is a demo video showing the steps to use this web application. Also please
|
||||
read
|
||||
[youtube8m/README.md](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/youtube8m/README.md)
|
||||
if you prefer to run the underlying model_inference binary in command line.
|
||||
|
||||
### Steps to run the YouTube-8M model inference graph with a local video
|
||||
|
||||
1. Make sure you have the features.pb from the feature extraction pipeline.
|
||||
|
||||
2. Copy the baseline model
|
||||
[(model card)](https://drive.google.com/file/d/1xTCi9-Nm9dt2KIk8WR0dDFrIssWawyXy/view)
|
||||
to local.
|
||||
|
||||
```bash
|
||||
curl -o /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz http://data.yt8m.org/models/baseline/saved_model.tar.gz
|
||||
|
||||
tar -xvf /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz -C /tmp/mediapipe
|
||||
```
|
||||
|
||||
3. Build and run the inference binary.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
|
||||
mediapipe/examples/desktop/youtube8m:model_inference
|
||||
|
||||
# segment_size is the number of seconds window of frames.
|
||||
# overlap is the number of seconds adjacent segments share.
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/model_inference \
|
||||
--calculator_graph_config_file=mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt \
|
||||
--input_side_packets=input_sequence_example_path=/tmp/mediapipe/features.pb,input_video_path=/absolute/path/to/the/local/video/file,output_video_path=/tmp/mediapipe/annotated_video.mp4,segment_size=5,overlap=4
|
||||
```
|
||||
|
||||
4. View the annotated video.
|
||||
Reference in New Issue
Block a user