Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 MediaPipe Teamandchuoling
|
f96eadd6df | ||
|
MediaPipe Teamandjqtang
|
a8d6ce95c4 | ||
|
MediaPipe Teamandchuoling
|
d53068fe2c | ||
|
MediaPipe Teamandchuoling
|
c828392681 | ||
|
MediaPipe Teamandchuoling
|
cccf6244d3 | ||
|
MediaPipe Teamandchuoling
|
8f69af91fe | ||
|
MediaPipe Teamandchuoling
|
a908d668c7 | ||
|
MediaPipe Teamandjqtang
|
1db91b550a | ||
|
MediaPipe Teamandchuoling
|
c0124fb83c | ||
|
MediaPipe Teamandchuoling
|
a7225b938a | ||
|
MediaPipe Teamandchuoling
|
73f4475c17 | ||
|
MediaPipe Teamandchuoling
|
f57ff46845 | ||
|
MediaPipe Teamandchuoling
|
d7c287c4e9 | ||
|
MediaPipe Teamandchuoling
|
6b0ab0e012 | ||
|
MediaPipe Teamandchuoling
|
2f86a459b6 |
+1
-1
@@ -12,6 +12,6 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
exports_files(["LICENSE"])
|
||||
+1
-1
@@ -54,7 +54,7 @@ RUN pip3 install tf_slim
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# Install bazel
|
||||
ARG BAZEL_VERSION=2.0.0
|
||||
ARG BAZEL_VERSION=3.4.1
|
||||
RUN mkdir /bazel && \
|
||||
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
|
||||
azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
global-exclude .git*
|
||||
global-exclude *_test.py
|
||||
|
||||
include CONTRIBUTING.md
|
||||
include LICENSE
|
||||
include MANIFEST.in
|
||||
include README.md
|
||||
include requirements.txt
|
||||
|
||||
recursive-include mediapipe/modules *.tflite *.txt *.binarypb
|
||||
@@ -8,44 +8,63 @@ nav_order: 1
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
## Cross-platform ML solutions made simple
|
||||
## Live ML anywhere
|
||||
|
||||
[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
|
||||
and developers to build world-class ML solutions and applications for mobile,
|
||||
desktop/cloud, web and IoT devices.
|
||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
||||
ML solutions for live and streaming media.
|
||||
|
||||
 | 
|
||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
||||
***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
 | 
|
||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
||||
|
||||
## ML solutions in MediaPipe
|
||||
|
||||
Face Detection | Face Mesh | Hands | Hair Segmentation
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :---------------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/hair_segmentation)
|
||||
Face Detection | Face Mesh | Iris | Hands | Pose | Hair Segmentation
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :---------------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/hair_segmentation)
|
||||
|
||||
Object Detection | Box Tracking | Objectron | KNIFT
|
||||
:----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
||||
:----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
||||
|
||||
[]() | Android | iOS | Desktop | Web | Coral
|
||||
:---------------------------------------------------------------------------- | :-----: | :-: | :-----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | |
|
||||
[]() | Android | iOS | Desktop | Python | Web | Coral
|
||||
:---------------------------------------------------------------------------------------- | :-----: | :-: | :-----: | :----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
|
||||
## MediaPipe in Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described in:
|
||||
|
||||
* [MediaPipe Face Mesh](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/face_mesh_py_colab)
|
||||
* [MediaPipe Hands](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/hands_py_colab)
|
||||
* [MediaPipe Pose](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/pose_py_colab)
|
||||
|
||||
## MediaPipe on the Web
|
||||
|
||||
@@ -63,8 +82,11 @@ never leaves your device.
|
||||
|
||||
|
||||
* [MediaPipe Face Detection](https://viz.mediapipe.dev/demo/face_detection)
|
||||
* [MediaPipe Iris](https://viz.mediapipe.dev/demo/iris_tracking)
|
||||
* [MediaPipe Iris: Depth-from-Iris](https://viz.mediapipe.dev/demo/iris_depth)
|
||||
* [MediaPipe Hands](https://viz.mediapipe.dev/demo/hand_tracking)
|
||||
* [MediaPipe Hands (palm/hand detection only)](https://viz.mediapipe.dev/demo/hand_detection)
|
||||
* [MediaPipe Pose](https://viz.mediapipe.dev/demo/pose_tracking)
|
||||
* [MediaPipe Hair Segmentation](https://viz.mediapipe.dev/demo/hair_segmentation)
|
||||
|
||||
## Getting started
|
||||
@@ -83,6 +105,14 @@ run code search using
|
||||
|
||||
## Publications
|
||||
|
||||
* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
|
||||
in Google Developers Blog
|
||||
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
|
||||
in Google Developers Blog
|
||||
* [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
in Google AI Blog
|
||||
* [MediaPipe Iris: Real-time Eye Tracking and Depth Estimation](https://ai.googleblog.com/2020/08/mediapipe-iris-real-time-iris-tracking.html)
|
||||
in Google AI Blog
|
||||
* [MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
|
||||
in Google Developers Blog
|
||||
* [Alfred Camera: Smart camera features using MediaPipe](https://developers.googleblog.com/2020/03/alfred-camera-smart-camera-features-using-mediapipe.html)
|
||||
|
||||
@@ -10,7 +10,7 @@ http_archive(
|
||||
sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
|
||||
)
|
||||
load("@bazel_skylib//lib:versions.bzl", "versions")
|
||||
versions.check(minimum_bazel_version = "2.0.0")
|
||||
versions.check(minimum_bazel_version = "3.4.0")
|
||||
|
||||
|
||||
# ABSL cpp library lts_2020_02_25
|
||||
@@ -36,6 +36,19 @@ http_archive(
|
||||
urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "rules_foreign_cc",
|
||||
strip_prefix = "rules_foreign_cc-master",
|
||||
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/master.zip",
|
||||
)
|
||||
|
||||
load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
|
||||
|
||||
rules_foreign_cc_dependencies()
|
||||
|
||||
# This is used to select all contents of the archives for CMake-based packages to give CMake access to them.
|
||||
all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])"""
|
||||
|
||||
# GoogleTest/GoogleMock framework. Used by most unit-tests.
|
||||
# Last updated 2020-06-30.
|
||||
http_archive(
|
||||
@@ -68,14 +81,23 @@ http_archive(
|
||||
url = "https://github.com/gflags/gflags/archive/v2.2.2.zip",
|
||||
)
|
||||
|
||||
# glog v0.3.5
|
||||
# TODO: Migrate MediaPipe to use com_github_glog_glog on all platforms.
|
||||
# 2020-08-21
|
||||
http_archive(
|
||||
name = "com_github_glog_glog_v_0_3_5",
|
||||
url = "https://github.com/google/glog/archive/v0.3.5.zip",
|
||||
sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
|
||||
strip_prefix = "glog-0.3.5",
|
||||
build_file = "@//third_party:glog.BUILD",
|
||||
name = "com_github_glog_glog",
|
||||
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
|
||||
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
|
||||
urls = [
|
||||
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
|
||||
],
|
||||
)
|
||||
http_archive(
|
||||
name = "com_github_glog_glog_no_gflags",
|
||||
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
|
||||
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
|
||||
build_file = "@//third_party:glog_no_gflags.BUILD",
|
||||
urls = [
|
||||
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
|
||||
],
|
||||
patches = [
|
||||
"@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
|
||||
],
|
||||
@@ -84,16 +106,6 @@ http_archive(
|
||||
],
|
||||
)
|
||||
|
||||
# 2020-02-16
|
||||
http_archive(
|
||||
name = "com_github_glog_glog",
|
||||
strip_prefix = "glog-3ba8976592274bc1f907c402ce22558011d6fc5e",
|
||||
sha256 = "feca3c7e29a693cab7887409756d89d342d4a992d54d7c5599bebeae8f7b50be",
|
||||
urls = [
|
||||
"https://github.com/google/glog/archive/3ba8976592274bc1f907c402ce22558011d6fc5e.zip",
|
||||
],
|
||||
)
|
||||
|
||||
# easyexif
|
||||
http_archive(
|
||||
name = "easyexif",
|
||||
@@ -137,6 +149,25 @@ http_archive(
|
||||
urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
|
||||
)
|
||||
|
||||
# 2020-07-09
|
||||
http_archive(
|
||||
name = "pybind11_bazel",
|
||||
strip_prefix = "pybind11_bazel-203508e14aab7309892a1c5f7dd05debda22d9a5",
|
||||
urls = ["https://github.com/pybind/pybind11_bazel/archive/203508e14aab7309892a1c5f7dd05debda22d9a5.zip"],
|
||||
sha256 = "75922da3a1bdb417d820398eb03d4e9bd067c4905a4246d35a44c01d62154d91",
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "pybind11",
|
||||
urls = [
|
||||
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.4.3.tar.gz",
|
||||
"https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz",
|
||||
],
|
||||
sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d",
|
||||
strip_prefix = "pybind11-2.4.3",
|
||||
build_file = "@pybind11_bazel//:pybind11.BUILD",
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "ceres_solver",
|
||||
url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
|
||||
@@ -150,6 +181,13 @@ http_archive(
|
||||
sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "opencv",
|
||||
build_file_content = all_content,
|
||||
strip_prefix = "opencv-3.4.10",
|
||||
urls = ["https://github.com/opencv/opencv/archive/3.4.10.tar.gz"],
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
@@ -165,13 +203,13 @@ new_local_repository(
|
||||
new_local_repository(
|
||||
name = "macos_opencv",
|
||||
build_file = "@//third_party:opencv_macos.BUILD",
|
||||
path = "/usr",
|
||||
path = "/usr/local/opt/opencv@3",
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "macos_ffmpeg",
|
||||
build_file = "@//third_party:ffmpeg_macos.BUILD",
|
||||
path = "/usr",
|
||||
path = "/usr/local/opt/ffmpeg",
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
@@ -282,24 +320,27 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
|
||||
maven_install(
|
||||
name = "maven",
|
||||
artifacts = [
|
||||
"junit:junit:4.12",
|
||||
"androidx.test.espresso:espresso-core:3.1.1",
|
||||
"org.hamcrest:hamcrest-library:1.3",
|
||||
"androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
"androidx.lifecycle:lifecycle-common:2.2.0",
|
||||
"androidx.annotation:annotation:aar:1.1.0",
|
||||
"androidx.appcompat:appcompat:aar:1.1.0-rc01",
|
||||
"androidx.camera:camera-core:aar:1.0.0-alpha06",
|
||||
"androidx.camera:camera-camera2:aar:1.0.0-alpha06",
|
||||
"androidx.camera:camera-core:1.0.0-beta10",
|
||||
"androidx.camera:camera-camera2:1.0.0-beta10",
|
||||
"androidx.camera:camera-lifecycle:1.0.0-beta10",
|
||||
"androidx.constraintlayout:constraintlayout:aar:1.1.3",
|
||||
"androidx.core:core:aar:1.1.0-rc03",
|
||||
"androidx.legacy:legacy-support-v4:aar:1.0.0",
|
||||
"androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
|
||||
"androidx.test.espresso:espresso-core:3.1.1",
|
||||
"com.github.bumptech.glide:glide:4.11.0",
|
||||
"com.google.android.material:material:aar:1.0.0-rc01",
|
||||
"com.google.code.findbugs:jsr305:3.0.2",
|
||||
"com.google.flogger:flogger-system-backend:0.3.1",
|
||||
"com.google.flogger:flogger:0.3.1",
|
||||
"com.google.guava:guava:27.0.1-android",
|
||||
"com.google.guava:listenablefuture:1.0",
|
||||
"junit:junit:4.12",
|
||||
"org.hamcrest:hamcrest-library:1.3",
|
||||
],
|
||||
repositories = [
|
||||
"https://jcenter.bintray.com",
|
||||
@@ -323,13 +364,12 @@ http_archive(
|
||||
)
|
||||
|
||||
#Tensorflow repo should always go after the other external dependencies.
|
||||
# 2020-05-11
|
||||
_TENSORFLOW_GIT_COMMIT = "7c09d15f9fcc14343343c247ebf5b8e0afe3e4aa"
|
||||
_TENSORFLOW_SHA256= "673d00cbd2676ae43df1993e0d28c10b5ffbe96d9e2ab29f88a77b43c0211299"
|
||||
# 2020-10-30
|
||||
_TENSORFLOW_GIT_COMMIT = "84384703c0d8b502e33ff6fd7eefd219dca5ff8e"
|
||||
_TENSORFLOW_SHA256= "23fb322fc15a20f7a7838d9a31f8b16f60700a494ea654311a0aa8621769df98"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
],
|
||||
patches = [
|
||||
|
||||
+28
-26
@@ -93,38 +93,40 @@ for app in ${apps}; do
|
||||
|
||||
echo "=== Target: ${target}"
|
||||
|
||||
if [[ $install_only == false ]]; then
|
||||
bazel_flags=("${default_bazel_flags[@]}")
|
||||
bazel_flags+=(${target})
|
||||
if [[ $strip == true ]]; then
|
||||
bazel_flags+=(--linkopt=-s)
|
||||
fi
|
||||
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_4
|
||||
fi
|
||||
bazel "${bazel_flags[@]}"
|
||||
cp -f "${bin}" "${apk}"
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_3
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${app_name} == "objectdetection3d" ]]; then
|
||||
orig_apk=${apk}
|
||||
apk="${out_dir}/${target_name}_shoes.apk"
|
||||
cp -f "${orig_apk}" "${apk}"
|
||||
apks+=(${apk})
|
||||
|
||||
apk="${out_dir}/${target_name}_chairs.apk"
|
||||
categories=("shoe" "chair" "cup" "camera" "shoe_1stage" "chair_1stage")
|
||||
for category in ${categories[@]}; do
|
||||
apk="${out_dir}/${target_name}_${category}.apk"
|
||||
if [[ $install_only == false ]]; then
|
||||
bazel_flags_extended=("${bazel_flags[@]}")
|
||||
if [[ ${category} != "shoe" ]]; then
|
||||
bazel_flags_extended+=(--define ${category}=true)
|
||||
fi
|
||||
echo "bazel ${bazel_flags_extended[@]}"
|
||||
bazel "${bazel_flags_extended[@]}"
|
||||
cp -f "${bin}" "${apk}"
|
||||
fi
|
||||
apks+=(${apk})
|
||||
done
|
||||
else
|
||||
if [[ $install_only == false ]]; then
|
||||
bazel_flags+=(--define chair=true)
|
||||
bazel_flags=("${default_bazel_flags[@]}")
|
||||
bazel_flags+=(${target})
|
||||
if [[ $strip == true ]]; then
|
||||
bazel_flags+=(--linkopt=-s)
|
||||
fi
|
||||
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_4
|
||||
fi
|
||||
bazel "${bazel_flags[@]}"
|
||||
cp -f "${bin}" "${apk}"
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_3
|
||||
fi
|
||||
fi
|
||||
apks+=(${apk})
|
||||
fi
|
||||
|
||||
apks+=(${apk})
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# =========================================================================
|
||||
#
|
||||
# Script to build/run all MediaPipe desktop example apps (with webcam input).
|
||||
#
|
||||
# To build and run all apps and store them in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir
|
||||
# Omitting -d and the associated directory saves all generated apps in the
|
||||
# current directory.
|
||||
# To build all apps and store them in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir -b
|
||||
# Omitting -d and the associated directory saves all generated apps in the
|
||||
# current directory.
|
||||
# To run all apps already stored in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir -r
|
||||
# Omitting -d and the associated directory assumes all apps are in the current
|
||||
# directory.
|
||||
|
||||
set -e
|
||||
|
||||
out_dir="."
|
||||
build_only=false
|
||||
run_only=false
|
||||
app_dir="mediapipe/examples/desktop"
|
||||
bin_dir="bazel-bin"
|
||||
declare -a default_bazel_flags=(build -c opt --define MEDIAPIPE_DISABLE_GPU=1)
|
||||
|
||||
while [[ -n $1 ]]; do
|
||||
case $1 in
|
||||
-d)
|
||||
shift
|
||||
out_dir=$1
|
||||
;;
|
||||
-b)
|
||||
build_only=true
|
||||
;;
|
||||
-r)
|
||||
run_only=true
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported input argument $1."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
echo "app_dir: $app_dir"
|
||||
echo "out_dir: $out_dir"
|
||||
|
||||
declare -a bazel_flags
|
||||
|
||||
apps="${app_dir}/*"
|
||||
for app in ${apps}; do
|
||||
if [[ -d "${app}" ]]; then
|
||||
target_name=${app##*/}
|
||||
if [[ "${target_name}" == "autoflip" ||
|
||||
"${target_name}" == "hello_world" ||
|
||||
"${target_name}" == "media_sequence" ||
|
||||
"${target_name}" == "template_matching" ||
|
||||
"${target_name}" == "youtube8m" ]]; then
|
||||
continue
|
||||
fi
|
||||
target="${app}:${target_name}_cpu"
|
||||
|
||||
echo "=== Target: ${target}"
|
||||
|
||||
if [[ $run_only == false ]]; then
|
||||
bazel_flags=("${default_bazel_flags[@]}")
|
||||
bazel_flags+=(${target})
|
||||
|
||||
bazel "${bazel_flags[@]}"
|
||||
cp -f "${bin_dir}/${app}/"*"_cpu" "${out_dir}"
|
||||
fi
|
||||
if [[ $build_only == false ]]; then
|
||||
if [[ ${target_name} == "object_tracking" ]]; then
|
||||
graph_name="tracking/object_detection_tracking"
|
||||
elif [[ ${target_name} == "upper_body_pose_tracking" ]]; then
|
||||
graph_name="pose_tracking/upper_body_pose_tracking"
|
||||
else
|
||||
graph_name="${target_name}/${target_name}"
|
||||
fi
|
||||
if [[ ${target_name} == "iris_tracking" ||
|
||||
${target_name} == "upper_body_pose_tracking" ]]; then
|
||||
graph_suffix="cpu"
|
||||
else
|
||||
graph_suffix="desktop_live"
|
||||
fi
|
||||
GLOG_logtostderr=1 "${out_dir}/${target_name}_cpu" \
|
||||
--calculator_graph_config_file=mediapipe/graphs/"${graph_name}_${graph_suffix}.pbtxt"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
@@ -58,6 +58,9 @@ apps="${app_dir}/*"
|
||||
for app in ${apps}; do
|
||||
if [[ -d "${app}" ]]; then
|
||||
target_name=${app##*/}
|
||||
if [[ "${target_name}" == "common" ]]; then
|
||||
continue
|
||||
fi
|
||||
target="${app}:${target_name}"
|
||||
|
||||
echo "=== Target: ${target}"
|
||||
|
||||
+2
-2
@@ -22,8 +22,8 @@ aux_links:
|
||||
# Footer content appears at the bottom of every page's main content
|
||||
footer_content: "© 2020 GOOGLE LLC | <a href=\"https://policies.google.com/privacy\">PRIVACY POLICY</a> | <a href=\"https://policies.google.com/terms\">TERMS OF SERVICE</a>"
|
||||
|
||||
# Color scheme currently only supports "dark" or nil (default)
|
||||
color_scheme: nil
|
||||
# Color scheme currently only supports "dark", "light"/nil (default), or a custom scheme that you define
|
||||
color_scheme: mediapipe
|
||||
|
||||
# Google Analytics Tracking (optional)
|
||||
ga_tracking: UA-140696581-2
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
$link-color: #0097A7;
|
||||
@@ -402,11 +402,6 @@ node {
|
||||
The diagram below shows how the `PacketClonerCalculator` defines its output
|
||||
packets (bottom) based on its series of input packets (top).
|
||||
|
||||
|  :
|
||||
| :--------------------------------------------------------------------------: |
|
||||
| *Each time it receives a packet on its TICK input stream, the |
|
||||
: PacketClonerCalculator outputs the most recent packet from each of its input :
|
||||
: streams. The sequence of output packets (bottom) is determined by the :
|
||||
: sequence of input packets (top) and their timestamps. The timestamps are :
|
||||
: shown along the right side of the diagram.* :
|
||||
|  |
|
||||
| :---------------------------------------------------------------------------: |
|
||||
| *Each time it receives a packet on its TICK input stream, the PacketClonerCalculator outputs the most recent packet from each of its input streams. The sequence of output packets (bottom) is determined by the sequence of input packets (top) and their timestamps. The timestamps are shown along the right side of the diagram.* |
|
||||
|
||||
@@ -20,7 +20,7 @@ Packets are generally created with `MediaPipe::Adopt()` (from packet.h).
|
||||
|
||||
```c++
|
||||
// Create some data.
|
||||
auto data = gtl::MakeUnique<MyDataClass>("constructor_argument");
|
||||
auto data = absl::make_unique<MyDataClass>("constructor_argument");
|
||||
// Create a packet to own the data.
|
||||
Packet p = Adopt(data.release());
|
||||
// Make a new packet with the same data and a different timestamp.
|
||||
|
||||
@@ -132,9 +132,10 @@ each project.
|
||||
implementation 'com.google.guava:guava:27.0.1-android'
|
||||
implementation 'com.google.protobuf:protobuf-java:3.11.4'
|
||||
// CameraX core library
|
||||
def camerax_version = "1.0.0-alpha06"
|
||||
def camerax_version = "1.0.0-beta10"
|
||||
implementation "androidx.camera:camera-core:$camerax_version"
|
||||
implementation "androidx.camera:camera-camera2:$camerax_version"
|
||||
implementation "androidx.camera:camera-lifecycle:$camerax_version"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -280,16 +280,16 @@ are two options:
|
||||
2. In the project navigator in the left sidebar, select the "Mediapipe"
|
||||
project.
|
||||
|
||||
3. Select the "Signing & Capabilities" tab.
|
||||
3. Select one of the application targets, e.g. HandTrackingGpuApp.
|
||||
|
||||
4. Select one of the application targets, e.g. HandTrackingGpuApp.
|
||||
4. Select the "Signing & Capabilities" tab.
|
||||
|
||||
5. Check "Automatically manage signing", and confirm the dialog box.
|
||||
|
||||
6. Select "_Your Name_ (Personal Team)" in the Team pop-up menu.
|
||||
|
||||
7. This set-up needs to be done once for each application you want to install.
|
||||
Repeat steps 4-6 as needed.
|
||||
Repeat steps 3-6 as needed.
|
||||
|
||||
This generates provisioning profiles for each app you have selected. Now we need
|
||||
to tell Bazel to use them. We have provided a script to make this easier.
|
||||
@@ -390,9 +390,6 @@ developer (yourself) is trusted.
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu
|
||||
```
|
||||
|
||||
This will open up your webcam as long as it is connected and on. Any errors
|
||||
is likely due to your webcam being not accessible.
|
||||
|
||||
2. To run the application:
|
||||
|
||||
```bash
|
||||
@@ -400,6 +397,9 @@ developer (yourself) is trusted.
|
||||
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt
|
||||
```
|
||||
|
||||
This will open up your webcam as long as it is connected and on. Any errors
|
||||
is likely due to your webcam being not accessible.
|
||||
|
||||
### Option 2: Running on GPU
|
||||
|
||||
Note: This currently works only on Linux, and please first follow
|
||||
@@ -412,13 +412,64 @@ Note: This currently works only on Linux, and please first follow
|
||||
mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu
|
||||
```
|
||||
|
||||
This will open up your webcam as long as it is connected and on. Any errors
|
||||
is likely due to your webcam being not accessible, or GPU drivers not setup
|
||||
properly.
|
||||
|
||||
2. To run the application:
|
||||
|
||||
```bash
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_gpu \
|
||||
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt
|
||||
```
|
||||
|
||||
This will open up your webcam as long as it is connected and on. Any errors
|
||||
is likely due to your webcam being not accessible, or GPU drivers not setup
|
||||
properly.
|
||||
|
||||
## Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described in, for instance,
|
||||
[Python section in MediaPipe Pose](../solutions/pose.md#python) and in this
|
||||
[colab](https://mediapipe.page.link/pose_py_colab).
|
||||
|
||||
Follow the steps below only if you have local changes and need to build the
|
||||
Python package from source. Otherwise, we strongly encourage our users to simply
|
||||
run `pip install mediapipe`, more convenient and much faster.
|
||||
|
||||
1. Make sure that Bazel and OpenCV are correctly installed and configured for
|
||||
MediaPipe. Please see [Installation](./install.md) for how to setup Bazel
|
||||
and OpenCV for MediaPipe on Linux and macOS.
|
||||
|
||||
2. Install the following dependencies.
|
||||
|
||||
```bash
|
||||
# Debian or Ubuntu
|
||||
$ sudo apt install python3-dev
|
||||
$ sudo apt install python3-venv
|
||||
$ sudo apt install -y protobuf-compiler
|
||||
```
|
||||
|
||||
```bash
|
||||
# macOS
|
||||
$ brew install protobuf
|
||||
```
|
||||
|
||||
3. Activate a Python virtual environment.
|
||||
|
||||
```bash
|
||||
$ python3 -m venv mp_env && source mp_env/bin/activate
|
||||
```
|
||||
|
||||
4. In the virtual environment, go to the MediaPipe repo directory.
|
||||
|
||||
5. Install the required Python packages.
|
||||
|
||||
```bash
|
||||
(mp_env)mediapipe$ pip3 install -r requirements.txt
|
||||
```
|
||||
|
||||
6. Generate and install MediaPipe package.
|
||||
|
||||
```bash
|
||||
(mp_env)mediapipe$ python3 setup.py gen_protos
|
||||
(mp_env)mediapipe$ python3 setup.py install --link-opencv
|
||||
```
|
||||
|
||||
@@ -446,8 +446,8 @@ visible so that we can start seeing frames from the `previewFrameTexture`.
|
||||
However, before starting the camera, we need to decide which camera we want to
|
||||
use. [`CameraXPreviewHelper`] inherits from [`CameraHelper`] which provides two
|
||||
options, `FRONT` and `BACK`. We can pass in the decision from the `BUILD` file
|
||||
as metadata such that no code change is required to build a another version of
|
||||
the app using a different camera.
|
||||
as metadata such that no code change is required to build another version of the
|
||||
app using a different camera.
|
||||
|
||||
Assuming we want to use `BACK` camera to perform edge detection on a live scene
|
||||
that we view from the camera, add the metadata into `AndroidManifest.xml`:
|
||||
|
||||
@@ -18,8 +18,8 @@ This codelab uses MediaPipe on an iOS device.
|
||||
|
||||
### What you will learn
|
||||
|
||||
How to develop an iOS application that uses MediaPipe and run a MediaPipe
|
||||
graph on iOS.
|
||||
How to develop an iOS application that uses MediaPipe and run a MediaPipe graph
|
||||
on iOS.
|
||||
|
||||
### What you will build
|
||||
|
||||
@@ -42,8 +42,8 @@ We will be using the following graph, [`edge_detection_mobile_gpu.pbtxt`]:
|
||||
```
|
||||
# MediaPipe graph that performs GPU Sobel edge detection on a live video stream.
|
||||
# Used in the examples
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/edgedetectiongpu.
|
||||
# mediapipe/examples/ios/edgedetectiongpu.
|
||||
# mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:helloworld
|
||||
# and mediapipe/examples/ios/helloworld.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
@@ -89,21 +89,21 @@ to build it.
|
||||
|
||||
First, create an XCode project via File > New > Single View App.
|
||||
|
||||
Set the product name to "EdgeDetectionGpu", and use an appropriate organization
|
||||
Set the product name to "HelloWorld", and use an appropriate organization
|
||||
identifier, such as `com.google.mediapipe`. The organization identifier
|
||||
alongwith the product name will be the `bundle_id` for the application, such as
|
||||
`com.google.mediapipe.EdgeDetectionGpu`.
|
||||
`com.google.mediapipe.HelloWorld`.
|
||||
|
||||
Set the language to Objective-C.
|
||||
|
||||
Save the project to an appropriate location. Let's call this
|
||||
`$PROJECT_TEMPLATE_LOC`. So your project will be in the
|
||||
`$PROJECT_TEMPLATE_LOC/EdgeDetectionGpu` directory. This directory will contain
|
||||
another directory named `EdgeDetectionGpu` and an `EdgeDetectionGpu.xcodeproj` file.
|
||||
`$PROJECT_TEMPLATE_LOC/HelloWorld` directory. This directory will contain
|
||||
another directory named `HelloWorld` and an `HelloWorld.xcodeproj` file.
|
||||
|
||||
The `EdgeDetectionGpu.xcodeproj` will not be useful for this tutorial, as we will
|
||||
use bazel to build the iOS application. The content of the
|
||||
`$PROJECT_TEMPLATE_LOC/EdgeDetectionGpu/EdgeDetectionGpu` directory is listed below:
|
||||
The `HelloWorld.xcodeproj` will not be useful for this tutorial, as we will use
|
||||
bazel to build the iOS application. The content of the
|
||||
`$PROJECT_TEMPLATE_LOC/HelloWorld/HelloWorld` directory is listed below:
|
||||
|
||||
1. `AppDelegate.h` and `AppDelegate.m`
|
||||
2. `ViewController.h` and `ViewController.m`
|
||||
@@ -112,10 +112,10 @@ use bazel to build the iOS application. The content of the
|
||||
5. `Main.storyboard` and `Launch.storyboard`
|
||||
6. `Assets.xcassets` directory.
|
||||
|
||||
Copy these files to a directory named `EdgeDetectionGpu` to a location that can
|
||||
access the MediaPipe source code. For example, the source code of the
|
||||
application that we will build in this tutorial is located in
|
||||
`mediapipe/examples/ios/EdgeDetectionGpu`. We will refer to this path as the
|
||||
Copy these files to a directory named `HelloWorld` to a location that can access
|
||||
the MediaPipe source code. For example, the source code of the application that
|
||||
we will build in this tutorial is located in
|
||||
`mediapipe/examples/ios/HelloWorld`. We will refer to this path as the
|
||||
`$APPLICATION_PATH` throughout the codelab.
|
||||
|
||||
Note: MediaPipe provides Objective-C bindings for iOS. The edge detection
|
||||
@@ -134,8 +134,8 @@ load(
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "EdgeDetectionGpuApp",
|
||||
bundle_id = "com.google.mediapipe.EdgeDetectionGpu",
|
||||
name = "HelloWorldApp",
|
||||
bundle_id = "com.google.mediapipe.HelloWorld",
|
||||
families = [
|
||||
"iphone",
|
||||
"ipad",
|
||||
@@ -143,11 +143,11 @@ ios_application(
|
||||
infoplists = ["Info.plist"],
|
||||
minimum_os_version = MIN_IOS_VERSION,
|
||||
provisioning_profile = "//mediapipe/examples/ios:developer_provisioning_profile",
|
||||
deps = [":EdgeDetectionGpuAppLibrary"],
|
||||
deps = [":HelloWorldAppLibrary"],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
name = "EdgeDetectionGpuAppLibrary",
|
||||
name = "HelloWorldAppLibrary",
|
||||
srcs = [
|
||||
"AppDelegate.m",
|
||||
"ViewController.m",
|
||||
@@ -172,9 +172,8 @@ The `objc_library` rule adds dependencies for the `AppDelegate` and
|
||||
`ViewController` classes, `main.m` and the application storyboards. The
|
||||
templated app depends only on the `UIKit` SDK.
|
||||
|
||||
The `ios_application` rule uses the `EdgeDetectionGpuAppLibrary` Objective-C
|
||||
library generated to build an iOS application for installation on your iOS
|
||||
device.
|
||||
The `ios_application` rule uses the `HelloWorldAppLibrary` Objective-C library
|
||||
generated to build an iOS application for installation on your iOS device.
|
||||
|
||||
Note: You need to point to your own iOS developer provisioning profile to be
|
||||
able to run the application on your iOS device.
|
||||
@@ -182,21 +181,20 @@ able to run the application on your iOS device.
|
||||
To build the app, use the following command in a terminal:
|
||||
|
||||
```
|
||||
bazel build -c opt --config=ios_arm64 <$APPLICATION_PATH>:EdgeDetectionGpuApp'
|
||||
bazel build -c opt --config=ios_arm64 <$APPLICATION_PATH>:HelloWorldApp'
|
||||
```
|
||||
|
||||
For example, to build the `EdgeDetectionGpuApp` application in
|
||||
`mediapipe/examples/ios/edgedetectiongpu`, use the following
|
||||
command:
|
||||
For example, to build the `HelloWorldApp` application in
|
||||
`mediapipe/examples/ios/helloworld`, use the following command:
|
||||
|
||||
```
|
||||
bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/edgedetectiongpu:EdgeDetectionGpuApp
|
||||
bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/helloworld:HelloWorldApp
|
||||
```
|
||||
|
||||
Then, go back to XCode, open Window > Devices and Simulators, select your
|
||||
device, and add the `.ipa` file generated by the command above to your device.
|
||||
Here is the document on [setting up and compiling](./building_examples.md#ios) iOS
|
||||
MediaPipe apps.
|
||||
Here is the document on [setting up and compiling](./building_examples.md#ios)
|
||||
iOS MediaPipe apps.
|
||||
|
||||
Open the application on your device. Since it is empty, it should display a
|
||||
blank white screen.
|
||||
@@ -502,8 +500,8 @@ in our app:
|
||||
}];
|
||||
```
|
||||
|
||||
Note: It is important to start the graph before starting the camera, so that
|
||||
the graph is ready to process frames as soon as the camera starts sending them.
|
||||
Note: It is important to start the graph before starting the camera, so that the
|
||||
graph is ready to process frames as soon as the camera starts sending them.
|
||||
|
||||
Earlier, when we received frames from the camera in the `processVideoFrame`
|
||||
function, we displayed them in the `_liveView` using the `_renderer`. Now, we
|
||||
@@ -552,9 +550,12 @@ results of running the edge detection graph on a live video feed. Congrats!
|
||||
|
||||
|
||||
|
||||
If you ran into any issues, please see the full code of the tutorial
|
||||
[here](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/edgedetectiongpu).
|
||||
Please note that the iOS examples now use a [common] template app. The code in
|
||||
this tutorial is used in the [common] template app. The [helloworld] app has the
|
||||
appropriate `BUILD` file dependencies for the edge detection graph.
|
||||
|
||||
[Bazel]:https://bazel.build/
|
||||
[`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt
|
||||
[`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/edge_detection/edge_detection_mobile_gpu.pbtxt
|
||||
[MediaPipe installation guide]:./install.md
|
||||
[common]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/common)
|
||||
[helloworld]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/helloworld)
|
||||
|
||||
@@ -12,7 +12,7 @@ nav_order: 1
|
||||
{:toc}
|
||||
---
|
||||
|
||||
Note: To interoperate with OpenCV, OpenCV 3.x and above are preferred. OpenCV
|
||||
Note: To interoperate with OpenCV, OpenCV 3.x to 4.1 are preferred. OpenCV
|
||||
2.x currently works but interoperability support may be deprecated in the
|
||||
future.
|
||||
|
||||
@@ -42,16 +42,16 @@ apps, see these [instructions](./building_examples.md#ios).
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
|
||||
to install Bazel 2.0 or higher.
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
For Nvidia Jetson and Raspberry Pi devices with ARM Ubuntu, Bazel needs to
|
||||
be built from source.
|
||||
|
||||
```bash
|
||||
# For Bazel 3.0.0
|
||||
wget https://github.com/bazelbuild/bazel/releases/download/3.0.0/bazel-3.0.0-dist.zip
|
||||
# For Bazel 3.4.0
|
||||
wget https://github.com/bazelbuild/bazel/releases/download/3.4.0/bazel-3.4.0-dist.zip
|
||||
sudo apt-get install build-essential openjdk-8-jdk python zip unzip
|
||||
unzip bazel-3.0.0-dist.zip
|
||||
unzip bazel-3.4.0-dist.zip
|
||||
env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
|
||||
sudo cp output/bazel /usr/local/bin/
|
||||
```
|
||||
@@ -221,7 +221,7 @@ build issues.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
|
||||
to install Bazel 2.0 or higher.
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
3. Install OpenCV.
|
||||
|
||||
@@ -356,7 +356,7 @@ build issues.
|
||||
|
||||
Option 2. Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
|
||||
to install Bazel 2.0 or higher.
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
4. Install OpenCV and FFmpeg.
|
||||
|
||||
@@ -427,7 +427,6 @@ build issues.
|
||||
linkstatic = 1,
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
```
|
||||
|
||||
5. Make sure that Python 3 and the Python "six" library are installed.
|
||||
@@ -506,7 +505,7 @@ next section.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
|
||||
to install Bazel 2.0 or higher.
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
6. Set Bazel variables.
|
||||
|
||||
@@ -567,7 +566,6 @@ next section.
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.280613 1200 hello_world.cc:56] Hello World!
|
||||
|
||||
```
|
||||
|
||||
If you run into a build error, please read
|
||||
@@ -607,14 +605,14 @@ cameras. Alternatively, you use a video file as input.
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
|
||||
https://storage.googleapis.com/bazel/3.0.0/release/bazel-3.0.0-installer-linux-x86_64.sh && \
|
||||
sudo mkdir -p /usr/local/bazel/3.0.0 && \
|
||||
chmod 755 bazel-3.0.0-installer-linux-x86_64.sh && \
|
||||
sudo ./bazel-3.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.0.0 && \
|
||||
source /usr/local/bazel/3.0.0/lib/bazel/bin/bazel-complete.bash
|
||||
https://storage.googleapis.com/bazel/3.4.0/release/bazel-3.4.0-installer-linux-x86_64.sh && \
|
||||
sudo mkdir -p /usr/local/bazel/3.4.0 && \
|
||||
chmod 755 bazel-3.4.0-installer-linux-x86_64.sh && \
|
||||
sudo ./bazel-3.4.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.4.0 && \
|
||||
source /usr/local/bazel/3.4.0/lib/bazel/bin/bazel-complete.bash
|
||||
|
||||
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.0.0/lib/bazel/bin/bazel version && \
|
||||
alias bazel='/usr/local/bazel/3.0.0/lib/bazel/bin/bazel'
|
||||
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.4.0/lib/bazel/bin/bazel version && \
|
||||
alias bazel='/usr/local/bazel/3.4.0/lib/bazel/bin/bazel'
|
||||
```
|
||||
|
||||
6. Checkout MediaPipe repository.
|
||||
|
||||
@@ -27,13 +27,14 @@ Repository command failed
|
||||
|
||||
usually indicates that Bazel fails to find the local Python binary. To solve
|
||||
this issue, please first find where the python binary is and then add
|
||||
`--action_env PYTHON_BIN_PATH=<path to python binary>` to the Bazel command like
|
||||
the following:
|
||||
`--action_env PYTHON_BIN_PATH=<path to python binary>` to the Bazel command. For
|
||||
example, you can switch to use the system default python3 binary by the
|
||||
following command:
|
||||
|
||||
```
|
||||
bazel build -c opt \
|
||||
--define MEDIAPIPE_DISABLE_GPU=1 \
|
||||
--action_env PYTHON_BIN_PATH="/path/to/python" \
|
||||
--action_env PYTHON_BIN_PATH=$(which python3) \
|
||||
mediapipe/examples/desktop/hello_world
|
||||
```
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 524 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 808 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 7.1 MiB After Width: | Height: | Size: 6.7 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 925 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.4 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 1019 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 10 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.8 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 182 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 889 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.5 MiB After Width: | Height: | Size: 923 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 670 KiB After Width: | Height: | Size: 744 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 625 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.8 MiB After Width: | Height: | Size: 968 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 996 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 313 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.9 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 224 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
+54
-24
@@ -8,44 +8,63 @@ nav_order: 1
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
## Cross-platform ML solutions made simple
|
||||
## Live ML anywhere
|
||||
|
||||
[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
|
||||
and developers to build world-class ML solutions and applications for mobile,
|
||||
desktop/cloud, web and IoT devices.
|
||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
||||
ML solutions for live and streaming media.
|
||||
|
||||
 | 
|
||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
||||
***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
 | 
|
||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
||||
|
||||
## ML solutions in MediaPipe
|
||||
|
||||
Face Detection | Face Mesh | Hands | Hair Segmentation
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :---------------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/hair_segmentation)
|
||||
Face Detection | Face Mesh | Iris | Hands | Pose | Hair Segmentation
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :---------------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/hair_segmentation)
|
||||
|
||||
Object Detection | Box Tracking | Objectron | KNIFT
|
||||
:----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
||||
:----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
||||
|
||||
[]() | Android | iOS | Desktop | Web | Coral
|
||||
:---------------------------------------------------------------------------- | :-----: | :-: | :-----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | |
|
||||
[]() | Android | iOS | Desktop | Python | Web | Coral
|
||||
:---------------------------------------------------------------------------------------- | :-----: | :-: | :-----: | :----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
|
||||
## MediaPipe in Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described in:
|
||||
|
||||
* [MediaPipe Face Mesh](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/face_mesh_py_colab)
|
||||
* [MediaPipe Hands](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/hands_py_colab)
|
||||
* [MediaPipe Pose](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/pose_py_colab)
|
||||
|
||||
## MediaPipe on the Web
|
||||
|
||||
@@ -63,8 +82,11 @@ never leaves your device.
|
||||
|
||||
|
||||
* [MediaPipe Face Detection](https://viz.mediapipe.dev/demo/face_detection)
|
||||
* [MediaPipe Iris](https://viz.mediapipe.dev/demo/iris_tracking)
|
||||
* [MediaPipe Iris: Depth-from-Iris](https://viz.mediapipe.dev/demo/iris_depth)
|
||||
* [MediaPipe Hands](https://viz.mediapipe.dev/demo/hand_tracking)
|
||||
* [MediaPipe Hands (palm/hand detection only)](https://viz.mediapipe.dev/demo/hand_detection)
|
||||
* [MediaPipe Pose](https://viz.mediapipe.dev/demo/pose_tracking)
|
||||
* [MediaPipe Hair Segmentation](https://viz.mediapipe.dev/demo/hair_segmentation)
|
||||
|
||||
## Getting started
|
||||
@@ -83,6 +105,14 @@ run code search using
|
||||
|
||||
## Publications
|
||||
|
||||
* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
|
||||
in Google Developers Blog
|
||||
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
|
||||
in Google Developers Blog
|
||||
* [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
in Google AI Blog
|
||||
* [MediaPipe Iris: Real-time Eye Tracking and Depth Estimation](https://ai.googleblog.com/2020/08/mediapipe-iris-real-time-iris-tracking.html)
|
||||
in Google AI Blog
|
||||
* [MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
|
||||
in Google Developers Blog
|
||||
* [Alfred Camera: Smart camera features using MediaPipe](https://developers.googleblog.com/2020/03/alfred-camera-smart-camera-features-using-mediapipe.html)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: AutoFlip (Saliency-aware Video Cropping)
|
||||
parent: Solutions
|
||||
nav_order: 9
|
||||
nav_order: 12
|
||||
---
|
||||
|
||||
# AutoFlip: Saliency-aware Video Cropping
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: Box Tracking
|
||||
parent: Solutions
|
||||
nav_order: 6
|
||||
nav_order: 8
|
||||
---
|
||||
|
||||
# MediaPipe Box Tracking
|
||||
|
||||
@@ -102,9 +102,4 @@ to cross-compile and run MediaPipe examples on the
|
||||
[BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs](https://arxiv.org/abs/1907.05047)
|
||||
([presentation](https://docs.google.com/presentation/d/1YCtASfnYyZtH-41QvnW5iZxELFnf0MF-pPWSLGj8yjQ/present?slide=id.g5bc8aeffdd_1_0))
|
||||
([poster](https://drive.google.com/file/d/1u6aB6wxDY7X2TmeUUKgFydulNtXkb3pu/view))
|
||||
* For front-facing/selfie camera:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
|
||||
[TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
|
||||
* For back-facing camera:
|
||||
[TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
|
||||
* [Model card](https://drive.google.com/file/d/1f39lSzU5Oq-j_OXgS67KfN5wNsoeAZ4V/view)
|
||||
* [Models and model cards](./models.md#face_detection)
|
||||
|
||||
+241
-20
@@ -19,13 +19,18 @@ landmarks in real-time even on mobile devices. It employs machine learning (ML)
|
||||
to infer the 3D surface geometry, requiring only a single camera input without
|
||||
the need for a dedicated depth sensor. Utilizing lightweight model architectures
|
||||
together with GPU acceleration throughout the pipeline, the solution delivers
|
||||
real-time performance critical for live experiences. The core of the solution is
|
||||
the same as what powers
|
||||
[YouTube Stories](https://youtube-creators.googleblog.com/2018/11/introducing-more-ways-to-share-your.html)'
|
||||
creator effects, the
|
||||
[Augmented Faces API in ARCore](https://developers.google.com/ar/develop/java/augmented-faces/)
|
||||
and the
|
||||
[ML Kit Face Contour Detection API](https://firebase.google.com/docs/ml-kit/face-detection-concepts#contours).
|
||||
real-time performance critical for live experiences.
|
||||
|
||||
Additionally, the solution is bundled with the Face Geometry module that bridges
|
||||
the gap between the face landmark estimation and useful real-time augmented
|
||||
reality (AR) applications. It establishes a metric 3D space and uses the face
|
||||
landmark screen positions to estimate face geometry within that space. The face
|
||||
geometry data consists of common 3D geometry primitives, including a face pose
|
||||
transformation matrix and a triangular face mesh. Under the hood, a lightweight
|
||||
statistical analysis method called
|
||||
[Procrustes Analysis](https://en.wikipedia.org/wiki/Procrustes_analysis) is
|
||||
employed to drive a robust, performant and portable logic. The analysis runs on
|
||||
CPU and has a minimal speed/memory footprint on top of the ML model inference.
|
||||
|
||||
 |
|
||||
:-------------------------------------------------------------: |
|
||||
@@ -67,15 +72,15 @@ Note: To visualize a graph, copy the graph and paste it into
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
## Models
|
||||
### Models
|
||||
|
||||
### Face Detection Model
|
||||
#### Face Detection Model
|
||||
|
||||
The face detector is the same [BlazeFace](https://arxiv.org/abs/1907.05047)
|
||||
model used in [MediaPipe Face Detection](./face_detection.md). Please refer to
|
||||
[MediaPipe Face Detection](./face_detection.md) for details.
|
||||
|
||||
### Face Landmark Model
|
||||
#### Face Landmark Model
|
||||
|
||||
For 3D face landmarks we employed transfer learning and trained a network with
|
||||
several objectives: the network simultaneously predicts 3D landmark coordinates
|
||||
@@ -98,7 +103,108 @@ You can find more information about the face landmark model in this
|
||||
|
||||
 |
|
||||
:------------------------------------------------------------------------: |
|
||||
*Fig 2. Output of MediaPipe Face Mesh: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
|
||||
*Fig 2. Face landmarks: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
|
||||
|
||||
## Face Geometry Module
|
||||
|
||||
The [Face Landmark Model](#face-landmark-model) performs a single-camera face landmark
|
||||
detection in the screen coordinate space: the X- and Y- coordinates are
|
||||
normalized screen coordinates, while the Z coordinate is relative and is scaled
|
||||
as the X coodinate under the
|
||||
[weak perspective projection camera model](https://en.wikipedia.org/wiki/3D_projection#Weak_perspective_projection).
|
||||
This format is well-suited for some applications, however it does not directly
|
||||
enable the full spectrum of augmented reality (AR) features like aligning a
|
||||
virtual 3D object with a detected face.
|
||||
|
||||
The
|
||||
[Face Geometry module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry)
|
||||
moves away from the screen coordinate space towards a metric 3D space and
|
||||
provides necessary primitives to handle a detected face as a regular 3D object.
|
||||
By design, you'll be able to use a perspective camera to project the final 3D
|
||||
scene back into the screen coordinate space with a guarantee that the face
|
||||
landmark positions are not changed.
|
||||
|
||||
### Key Concepts
|
||||
|
||||
#### Metric 3D Space
|
||||
|
||||
The **Metric 3D space** established within the Face Geometry module is a
|
||||
right-handed orthonormal metric 3D coordinate space. Within the space, there is
|
||||
a **virtual perspective camera** located at the space origin and pointed in the
|
||||
negative direction of the Z-axis. In the current pipeline, it is assumed that
|
||||
the input camera frames are observed by exactly this virtual camera and
|
||||
therefore its parameters are later used to convert the screen landmark
|
||||
coordinates back into the Metric 3D space. The *virtual camera parameters* can
|
||||
be set freely, however for better results it is advised to set them as close to
|
||||
the *real physical camera parameters* as possible.
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------: |
|
||||
*Fig 3. A visualization of multiple key elements in the Metric 3D space.* |
|
||||
|
||||
#### Canonical Face Model
|
||||
|
||||
The **Canonical Face Model** is a static 3D model of a human face, which follows
|
||||
the 468 3D face landmark topology of the
|
||||
[Face Landmark Model](#face-landmark-model). The model bears two important
|
||||
functions:
|
||||
|
||||
- **Defines metric units**: the scale of the canonical face model defines the
|
||||
metric units of the Metric 3D space. A metric unit used by the
|
||||
[default canonical face model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.fbx)
|
||||
is a centimeter;
|
||||
- **Bridges static and runtime spaces**: the face pose transformation matrix
|
||||
is - in fact - a linear map from the canonical face model into the runtime
|
||||
face landmark set estimated on each frame. This way, virtual 3D assets
|
||||
modeled around the canonical face model can be aligned with a tracked face
|
||||
by applying the face pose transformation matrix to them.
|
||||
|
||||
### Components
|
||||
|
||||
#### Geometry Pipeline
|
||||
|
||||
The **Geometry Pipeline** is a key component, which is responsible for
|
||||
estimating face geometry objects within the Metric 3D space. On each frame, the
|
||||
following steps are executed in the given order:
|
||||
|
||||
- Face landmark screen coordinates are converted into the Metric 3D space
|
||||
coordinates;
|
||||
- Face pose transformation matrix is estimated as a rigid linear mapping from
|
||||
the canonical face metric landmark set into the runtime face metric landmark
|
||||
set in a way that minimizes a difference between the two;
|
||||
- A face mesh is created using the runtime face metric landmarks as the vertex
|
||||
positions (XYZ), while both the vertex texture coordinates (UV) and the
|
||||
triangular topology are inherited from the canonical face model.
|
||||
|
||||
The geometry pipeline is implemented as a MediaPipe
|
||||
[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc).
|
||||
For your convenience, the face geometry pipeline calculator is bundled together
|
||||
with the face landmark module into a unified MediaPipe
|
||||
[subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_front_gpu.pbtxt).
|
||||
The face geometry format is defined as a Protocol Buffer
|
||||
[message](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto).
|
||||
|
||||
#### Effect Renderer
|
||||
|
||||
The **Effect Renderer** is a component, which serves as a working example of a
|
||||
face effect renderer. It targets the *OpenGL ES 2.0* API to enable a real-time
|
||||
performance on mobile devices and supports the following rendering modes:
|
||||
|
||||
- **3D object rendering mode**: a virtual object is aligned with a detected
|
||||
face to emulate an object attached to the face (example: glasses);
|
||||
- **Face mesh rendering mode**: a texture is stretched on top of the face mesh
|
||||
surface to emulate a face painting technique.
|
||||
|
||||
In both rendering modes, the face mesh is first rendered as an occluder straight
|
||||
into the depth buffer. This step helps to create a more believable effect via
|
||||
hiding invisible elements behind the face surface.
|
||||
|
||||
The effect renderer is implemented as a MediaPipe
|
||||
[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/effect_renderer_calculator.cc).
|
||||
|
||||
|  |
|
||||
| :---------------------------------------------------------------------: |
|
||||
| *Fig 4. An example of face effects rendered by the Face Geometry Effect Renderer.* |
|
||||
|
||||
## Example Apps
|
||||
|
||||
@@ -111,7 +217,12 @@ Note: To visualize a graph, copy the graph and paste it into
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
### Mobile
|
||||
### Face Landmark Example
|
||||
|
||||
Face landmark example showcases real-time, cross-platform face landmark
|
||||
detection. For visual reference, please refer to *Fig. 2*.
|
||||
|
||||
#### Mobile
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
|
||||
@@ -125,9 +236,9 @@ Tip: Maximum number of faces to detect/process is set to 1 by default. To change
|
||||
it, for Android modify `NUM_FACES` in
|
||||
[MainActivity.java](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/MainActivity.java),
|
||||
and for iOS modify `kNumFaces` in
|
||||
[ViewController.mm](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facemeshgpu/ViewController.mm).
|
||||
[FaceMeshGpuViewController.mm](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.mm).
|
||||
|
||||
### Desktop
|
||||
#### Desktop
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
@@ -143,18 +254,128 @@ and for iOS modify `kNumFaces` in
|
||||
Tip: Maximum number of faces to detect/process is set to 1 by default. To change
|
||||
it, in the graph file modify the option of `ConstantSidePacketCalculator`.
|
||||
|
||||
#### Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described below and in this
|
||||
[colab](https://mediapipe.page.link/face_mesh_py_colab). If you do need to build
|
||||
the Python package from source, see
|
||||
[additional instructions](../getting_started/building_examples.md#python).
|
||||
|
||||
Activate a Python virtual environment:
|
||||
|
||||
```bash
|
||||
$ python3 -m venv mp_env && source mp_env/bin/activate
|
||||
```
|
||||
|
||||
Install MediaPipe Python package:
|
||||
|
||||
```bash
|
||||
(mp_env)$ pip install mediapipe
|
||||
```
|
||||
|
||||
Run the following Python code:
|
||||
|
||||
<!-- Do not change the example code below directly. Change the corresponding example in mediapipe/python/solutions/face_mesh.py and copy it over. -->
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_face_mesh = mp.solutions.face_mesh
|
||||
|
||||
# For static images:
|
||||
face_mesh = mp_face_mesh.FaceMesh(
|
||||
static_image_mode=True,
|
||||
max_num_faces=1,
|
||||
min_detection_confidence=0.5)
|
||||
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Print and draw face mesh landmarks on the image.
|
||||
if not results.multi_face_landmarks:
|
||||
continue
|
||||
annotated_image = image.copy()
|
||||
for face_landmarks in results.multi_face_landmarks:
|
||||
print('face_landmarks:', face_landmarks)
|
||||
mp_drawing.draw_landmarks(
|
||||
image=annotated_image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACE_CONNECTIONS,
|
||||
landmark_drawing_spec=drawing_spec,
|
||||
connection_drawing_spec=drawing_spec)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', image)
|
||||
face_mesh.close()
|
||||
|
||||
# For webcam input:
|
||||
face_mesh = mp_face_mesh.FaceMesh(
|
||||
min_detection_confidence=0.5, min_tracking_confidence=0.5)
|
||||
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = face_mesh.process(image)
|
||||
|
||||
# Draw the face mesh annotations on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.multi_face_landmarks:
|
||||
for face_landmarks in results.multi_face_landmarks:
|
||||
mp_drawing.draw_landmarks(
|
||||
image=image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACE_CONNECTIONS,
|
||||
landmark_drawing_spec=drawing_spec,
|
||||
connection_drawing_spec=drawing_spec)
|
||||
cv2.imshow('MediaPipe FaceMesh', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
face_mesh.close()
|
||||
cap.release()
|
||||
```
|
||||
|
||||
### Face Effect Example
|
||||
|
||||
Face effect example showcases real-time mobile face effect application use case
|
||||
for the Face Mesh solution. To enable a better user experience, this example
|
||||
only works for a single face. For visual reference, please refer to *Fig. 4*.
|
||||
|
||||
#### Mobile
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_effect/face_effect_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_effect/face_effect_gpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1ccnaDnffEuIXriBZr2SK_Eu4FpO7K44s)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/faceeffect`](http:/mediapipe/examples/ios/faceeffect/BUILD)
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
[Real-Time AR Self-Expression with Machine Learning](https://ai.googleblog.com/2019/03/real-time-ar-self-expression-with.html)
|
||||
* TensorFlow Blog:
|
||||
[Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
|
||||
* Google Developers Blog:
|
||||
[MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
|
||||
* Paper:
|
||||
[Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724)
|
||||
([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212))
|
||||
* Face detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite)
|
||||
* Face landmark mode:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
|
||||
* [Model card](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view)
|
||||
* Canonical face model:
|
||||
[FBX](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.fbx),
|
||||
[OBJ](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.obj),
|
||||
[UV visualization](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png)
|
||||
* [Models and model cards](./models.md#face_mesh)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: Hair Segmentation
|
||||
parent: Solutions
|
||||
nav_order: 4
|
||||
nav_order: 6
|
||||
---
|
||||
|
||||
# MediaPipe Hair Segmentation
|
||||
@@ -54,5 +54,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
[Real-time Hair segmentation and recoloring on Mobile GPUs](https://arxiv.org/abs/1907.06740)
|
||||
([presentation](https://drive.google.com/file/d/1C8WYlWdDRNtU1_pYBvkkG5Z5wqYqf0yj/view))
|
||||
([supplementary video](https://drive.google.com/file/d/1LPtM99Ch2ogyXYbDNpEqnUfhFq0TfLuf/view))
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
|
||||
* [Model card](https://drive.google.com/file/d/1lPwJ8BD_-3UUor4LayQ0xpa_RIC_hoRh/view)
|
||||
* [Models and model cards](./models.md#hair_segmentation)
|
||||
|
||||
+113
-57
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: Hands
|
||||
parent: Solutions
|
||||
nav_order: 3
|
||||
nav_order: 4
|
||||
---
|
||||
|
||||
# MediaPipe Hands
|
||||
@@ -55,13 +55,21 @@ frame, and only when the landmark model could no longer identify hand presence
|
||||
is palm detection invoked to relocalize the hand.
|
||||
|
||||
The pipeline is implemented as a MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt),
|
||||
which internally utilizes a
|
||||
[palm/hand detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_detection_gpu.pbtxt),
|
||||
a
|
||||
[hand landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt)
|
||||
and a
|
||||
[renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt).
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt)
|
||||
that uses a
|
||||
[hand landmark tracking subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu.pbtxt)
|
||||
from the
|
||||
[hand landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark),
|
||||
and renders using a dedicated
|
||||
[hand renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_renderer_gpu.pbtxt).
|
||||
The
|
||||
[hand landmark tracking subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu.pbtxt)
|
||||
internally uses a
|
||||
[hand landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_gpu.pbtxt)
|
||||
from the same module and a
|
||||
[palm detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_gpu.pbtxt)
|
||||
from the
|
||||
[palm detection module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection).
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
@@ -146,34 +154,11 @@ to visualize its associated subgraphs, please see
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handtrackinggpu/BUILD)
|
||||
|
||||
#### With Multi-hand Support
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1Wk6V9EVaz1ks_MInPqqVGvvJD01SGXDc)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu:multihandtrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/multihandtrackinggpu/BUILD)
|
||||
|
||||
There are two key differences between this graph and that in the
|
||||
[main example](#main-example) (which handles only one hand):
|
||||
|
||||
1. There is a `NormalizedRectVectorHasMinSize` calculator, that checks if in
|
||||
input vector of `NormalizedRect` objects has a minimum size equal to `N`. In
|
||||
this graph, if the vector contains fewer than `N` objects,
|
||||
`MultiHandDetection` subgraph runs. Otherwise, the `GateCalculator` doesn't
|
||||
send any image packets to the `MultiHandDetection` subgraph. This way, the
|
||||
main graph is efficient in that it avoids running the costly hand detection
|
||||
step when there are already `N` hands in the frame.
|
||||
2. The `MergeCalculator` has been replaced by the `AssociationNormRect`
|
||||
calculator. This `AssociationNormRect` takes as input a vector of
|
||||
`NormalizedRect` objects from the `MultiHandDetection` subgraph on the
|
||||
current frame, and a vector of `NormalizedRect` objects from the
|
||||
`MultiHandLandmark` subgraph from the previous frame, and performs an
|
||||
association operation between these objects. This calculator ensures that
|
||||
the output vector doesn't contain overlapping regions based on the specified
|
||||
`min_similarity_threshold`.
|
||||
Tip: Maximum number of hands to detect/process is set to 2 by default. To change
|
||||
it, for Android modify `NUM_HANDS` in
|
||||
[MainActivity.java](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java),
|
||||
and for iOS modify `kNumHands` in
|
||||
[HandTrackingViewController.mm](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handtrackinggpu/HandTrackingViewController.mm).
|
||||
|
||||
#### Palm/Hand Detection Only (no landmarks)
|
||||
|
||||
@@ -187,8 +172,6 @@ There are two key differences between this graph and that in the
|
||||
|
||||
### Desktop
|
||||
|
||||
#### Main Example
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt)
|
||||
@@ -196,22 +179,101 @@ There are two key differences between this graph and that in the
|
||||
[`mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/hand_tracking/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt)
|
||||
[`mediapipe/graphs/hand_tracking/hand_tracking_desktop_live_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_desktop_gpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/hand_tracking/BUILD)
|
||||
|
||||
#### With Multi-hand Support
|
||||
Tip: Maximum number of hands to detect/process is set to 2 by default. To change
|
||||
it, in the graph file modify the option of `ConstantSidePacketCalculator`.
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/multi_hand_tracking/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/multi_hand_tracking/BUILD)
|
||||
### Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described below and in this
|
||||
[colab](https://mediapipe.page.link/hands_py_colab). If you do need to build the
|
||||
Python package from source, see
|
||||
[additional instructions](../getting_started/building_examples.md#python).
|
||||
|
||||
Activate a Python virtual environment:
|
||||
|
||||
```bash
|
||||
$ python3 -m venv mp_env && source mp_env/bin/activate
|
||||
```
|
||||
|
||||
Install MediaPipe Python package:
|
||||
|
||||
```bash
|
||||
(mp_env)$ pip install mediapipe
|
||||
```
|
||||
|
||||
Run the following Python code:
|
||||
|
||||
<!-- Do not change the example code below directly. Change the corresponding example in mediapipe/python/solutions/hands.py and copy it over. -->
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_hands = mp.solutions.hands
|
||||
|
||||
# For static images:
|
||||
hands = mp_hands.Hands(
|
||||
static_image_mode=True,
|
||||
max_num_hands=2,
|
||||
min_detection_confidence=0.7)
|
||||
for idx, file in enumerate(file_list):
|
||||
# Read an image, flip it around y-axis for correct handedness output (see
|
||||
# above).
|
||||
image = cv2.flip(cv2.imread(file), 1)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Print handedness and draw hand landmarks on the image.
|
||||
print('handedness:', results.multi_handedness)
|
||||
if not results.multi_hand_landmarks:
|
||||
continue
|
||||
annotated_image = image.copy()
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
print('hand_landmarks:', hand_landmarks)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
cv2.imwrite(
|
||||
'/tmp/annotated_image' + str(idx) + '.png', cv2.flip(image, 1))
|
||||
hands.close()
|
||||
|
||||
# For webcam input:
|
||||
hands = mp_hands.Hands(
|
||||
min_detection_confidence=0.7, min_tracking_confidence=0.5)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = hands.process(image)
|
||||
|
||||
# Draw the hand annotations on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.multi_hand_landmarks:
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
mp_drawing.draw_landmarks(
|
||||
image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Hands', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
hands.close()
|
||||
cap.release()
|
||||
```
|
||||
|
||||
Tip: Use command `deactivate` to exit the Python virtual environment.
|
||||
|
||||
### Web
|
||||
|
||||
@@ -226,10 +288,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
* Paper:
|
||||
[MediaPipe Hands: On-device Real-time Hand Tracking](https://arxiv.org/abs/2006.10214)
|
||||
([presentation](https://www.youtube.com/watch?v=I-UOrvxxXEk))
|
||||
* Palm detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
|
||||
* Hand landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
|
||||
* [Model card](https://mediapipe.page.link/handmc)
|
||||
* [Models and model cards](./models.md#hands)
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
---
|
||||
layout: default
|
||||
title: Instant Motion Tracking
|
||||
parent: Solutions
|
||||
nav_order: 9
|
||||
---
|
||||
|
||||
# MediaPipe Instant Motion Tracking
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Augmented Reality (AR) technology creates fun, engaging, and immersive user
|
||||
experiences. The ability to perform AR tracking across devices and platforms,
|
||||
without initialization, remains important to power AR applications at scale.
|
||||
|
||||
MediaPipe Instant Motion Tracking provides AR tracking across devices and
|
||||
platforms without initialization or calibration. It is built upon the
|
||||
[MediaPipe Box Tracking](./box_tracking.md) solution. With Instant Motion
|
||||
Tracking, you can easily place virtual 2D and 3D content on static or moving
|
||||
surfaces, allowing them to seamlessly interact with the real-world environment.
|
||||
|
||||
 |
|
||||
:-----------------------------------------------------------------------: |
|
||||
*Fig 1. Instant Motion Tracking is used to augment the world with a 3D sticker.* |
|
||||
|
||||
## Pipeline
|
||||
|
||||
The Instant Motion Tracking pipeline is implemented as a MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt),
|
||||
which internally utilizes a
|
||||
[RegionTrackingSubgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/subgraphs/region_tracking.pbtxt)
|
||||
in order to perform anchor tracking for each individual 3D sticker.
|
||||
|
||||
We first use a
|
||||
[StickerManagerCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/calculators/sticker_manager_calculator.cc)
|
||||
to prepare the individual sticker data for the rest of the application. This
|
||||
information is then sent to the
|
||||
[RegionTrackingSubgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/subgraphs/region_tracking.pbtxt)
|
||||
that performs 3D region tracking for sticker placement and rendering. Once
|
||||
acquired, our tracked sticker regions are sent with user transformations (i.e.
|
||||
gestures from the user to rotate and zoom the sticker) and IMU data to the
|
||||
[MatricesManagerCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc),
|
||||
which turns all our sticker transformation data into a set of model matrices.
|
||||
This data is handled directly by our
|
||||
[GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc)
|
||||
as an input stream, which will render the provided texture and object file using
|
||||
our matrix specifications. The output of
|
||||
[GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc)
|
||||
is a video stream depicting the virtual 3D content rendered on top of the real
|
||||
world, creating immersive AR experiences for users.
|
||||
|
||||
## Using Instant Motion Tracking
|
||||
|
||||
With the Instant Motion Tracking MediaPipe [graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt),
|
||||
an application can create an interactive and realistic AR experience by
|
||||
specifying the required input streams, side packets, and output streams.
|
||||
The input streams are the following:
|
||||
|
||||
* Input Video (GpuBuffer): Video frames to render augmented stickers onto.
|
||||
* Rotation Matrix (9-element Float Array): The 3x3 row-major rotation
|
||||
matrix from the device IMU to determine proper orientation of the device.
|
||||
* Sticker Proto String (String): A string representing the
|
||||
serialized [sticker buffer protobuf message](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/calculators/sticker_buffer.proto),
|
||||
containing a list of all stickers and their attributes.
|
||||
* Each sticker in the Protobuffer has a unique ID to find associated
|
||||
anchors and transforms, an initial anchor placement in a normalized [0.0, 1.0]
|
||||
3D space, a user rotation and user scaling transform on the sticker,
|
||||
and an integer indicating which type of objects to render for the
|
||||
sticker (e.g. 3D asset or GIF).
|
||||
* Sticker Sentinel (Integer): When an anchor must be initially placed or
|
||||
repositioned, this value must be changed to the ID of the anchor to reset from
|
||||
the sticker buffer protobuf message. If no valid ID is provided, the system
|
||||
will simply maintain tracking.
|
||||
|
||||
Side packets are also an integral part of the Instant Motion Tracking solution
|
||||
to provide device-specific information for the rendering system:
|
||||
|
||||
* Field of View (Float): The field of view of the camera in radians.
|
||||
* Aspect Ratio (Float): The aspect ratio (width / height) of the camera frames
|
||||
(this ratio corresponds to the image frames themselves, not necessarily the
|
||||
screen bounds).
|
||||
* Object Asset (String): The
|
||||
[GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc)
|
||||
must be provided with an associated asset file name pointing to the 3D model
|
||||
to render in the viewfinder.
|
||||
* (Optional) Texture (ImageFrame on Android, GpuBuffer on iOS): Textures for
|
||||
the
|
||||
[GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc)
|
||||
can be provided either via an input stream (dynamic texturing) or as a side
|
||||
packet (unchanging texture).
|
||||
|
||||
The rendering system for the Instant Motion Tracking is powered by OpenGL. For
|
||||
more information regarding the structure of model matrices and OpenGL rendering,
|
||||
please visit [OpenGL Wiki](https://www.khronos.org/opengl/wiki/). With the
|
||||
specifications above, the Instant Motion Tracking capabilities can be adapted to
|
||||
any device that is able to run the MediaPipe framework with a working IMU system
|
||||
and connected camera.
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android) on how to build
|
||||
MediaPipe examples.
|
||||
|
||||
* Graph: [mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt)
|
||||
|
||||
* Android target (or download prebuilt [ARM64 APK](https://drive.google.com/file/d/1KnaBBoKpCHR73nOBJ4fL_YdWVTAcwe6L/view?usp=sharing)):
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking:instantmotiontracking`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD)
|
||||
|
||||
## Resources
|
||||
|
||||
* Google Developers Blog:
|
||||
[Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
|
||||
* Google AI Blog:
|
||||
[The Instant Motion Tracking Behind Motion Stills AR](https://ai.googleblog.com/2018/02/the-instant-motion-tracking-behind.html)
|
||||
* Paper:
|
||||
[Instant Motion Tracking and Its Applications to Augmented Reality](https://arxiv.org/abs/1907.06796)
|
||||
@@ -0,0 +1,202 @@
|
||||
---
|
||||
layout: default
|
||||
title: Iris
|
||||
parent: Solutions
|
||||
nav_order: 3
|
||||
---
|
||||
|
||||
# MediaPipe Iris
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
A wide range of real-world applications, including computational photography
|
||||
(glint reflection) and augmented reality effects (virtual avatars) rely on
|
||||
accurately tracking the iris within an eye. This is a challenging task to solve
|
||||
on mobile devices, due to the limited computing resources, variable light
|
||||
conditions and the presence of occlusions, such as hair or people squinting.
|
||||
Iris tracking can also be utilized to determine the metric distance of the
|
||||
camera to the user. This can improve a variety of use cases, ranging from
|
||||
virtual try-on of properly sized glasses and hats to accessibility features that
|
||||
adopt the font size depending on the viewer’s distance. Often, sophisticated
|
||||
specialized hardware is employed to compute the metric distance, limiting the
|
||||
range of devices on which the solution could be applied.
|
||||
|
||||
MediaPipe Iris is a ML solution for accurate iris estimation, able to track
|
||||
landmarks involving the iris, pupil and the eye contours using a single RGB
|
||||
camera, in real-time, without the need for specialized hardware. Through use of
|
||||
iris landmarks, the solution is also able to determine the metric distance
|
||||
between the subject and the camera with relative error less than 10%. Note that
|
||||
iris tracking does not infer the location at which people are looking, nor does
|
||||
it provide any form of identity recognition. With the cross-platfrom capability
|
||||
of the MediaPipe framework, MediaPipe Iris can run on most modern
|
||||
[mobile phones](#mobile), [desktops/laptops](#desktop) and even on the
|
||||
[web](#web).
|
||||
|
||||
 |
|
||||
:------------------------------------------------------------------------: |
|
||||
*Fig 1. Example of MediaPipe Iris: eyelid (red) and iris (blue) contours.* |
|
||||
|
||||
## ML Pipeline
|
||||
|
||||
The first step in the pipeline leverages [MediaPipe Face Mesh](./face_mesh.md),
|
||||
which generates a mesh of the approximate face geometry. From this mesh, we
|
||||
isolate the eye region in the original image for use in the subsequent iris
|
||||
tracking step.
|
||||
|
||||
The pipeline is implemented as a MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/iris_tracking/iris_tracking_gpu.pbtxt)
|
||||
that uses a
|
||||
[face landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
|
||||
from the
|
||||
[face landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark),
|
||||
an
|
||||
[iris landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark_left_and_right_gpu.pbtxt)
|
||||
from the
|
||||
[iris landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark),
|
||||
and renders using a dedicated
|
||||
[iris-and-depth renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/iris_tracking/subgraphs/iris_and_depth_renderer_gpu.pbtxt).
|
||||
The
|
||||
[face landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
|
||||
internally uses a
|
||||
[face detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt)
|
||||
from the
|
||||
[face detection module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection).
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
The output of the pipeline is a set of 478 3D landmarks, including 468 face
|
||||
landmarks from [MediaPipe Face Mesh](./face_mesh.md), with those around the eyes
|
||||
further refined (see Fig 2), and 10 additional iris landmarks appended at the
|
||||
end (5 for each eye, and see Fig 2 also).
|
||||
|
||||
## Models
|
||||
|
||||
### Face Detection Model
|
||||
|
||||
The face detector is the same [BlazeFace](https://arxiv.org/abs/1907.05047)
|
||||
model used in [MediaPipe Face Detection](./face_detection.md).
|
||||
|
||||
### Face Landmark Model
|
||||
|
||||
The face landmark model is the same as in [MediaPipe Face Mesh](./face_mesh.md).
|
||||
You can also find more details in this
|
||||
[paper](https://arxiv.org/abs/1907.06724).
|
||||
|
||||
### Iris Landmark Model
|
||||
|
||||
The iris model takes an image patch of the eye region and estimates both the eye
|
||||
landmarks (along the eyelid) and iris landmarks (along ths iris contour). You
|
||||
can find more details in this [paper](https://arxiv.org/abs/2006.11341).
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------------------------------: |
|
||||
*Fig 2. Eye landmarks (red) and iris landmarks (green).* |
|
||||
|
||||
## Depth-from-Iris
|
||||
|
||||
MediaPipe Iris is able to determine the metric distance of a subject to the
|
||||
camera with less than 10% error, without requiring any specialized hardware.
|
||||
This is done by relying on the fact that the horizontal iris diameter of the
|
||||
human eye remains roughly constant at 11.7±0.5 mm across a wide population,
|
||||
along with some simple geometric arguments. For more details please refer to our
|
||||
[Google AI Blog post](https://ai.googleblog.com/2020/08/mediapipe-iris-real-time-iris-tracking.html).
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------: |
|
||||
*Fig 3. (Left) MediaPipe Iris predicting metric distance in cm on a Pixel 2 from iris tracking without use of a depth sensor. (Right) Ground-truth depth.* |
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android),
|
||||
[iOS](../getting_started/building_examples.md#ios) and
|
||||
[desktop](../getting_started/building_examples.md#desktop) on how to build
|
||||
MediaPipe examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
### Mobile
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/iris_tracking/iris_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/iris_tracking/iris_tracking_gpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1cywcNtqk764TlZf1lvSTV4F3NGB2aL1R/view?usp=sharing)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu:iristrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/iristrackinggpu:IrisTrackingGpuApp`](http:/mediapipe/examples/ios/iristrackinggpu/BUILD)
|
||||
|
||||
### Desktop
|
||||
|
||||
#### Live Camera Input
|
||||
|
||||
Please first see general instructions for
|
||||
[desktop](../getting_started/building_examples.md#desktop) on how to build
|
||||
MediaPipe examples.
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/iris_tracking/iris_tracking_cpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/iris_tracking/iris_tracking_cpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/iris_tracking:iris_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/iris_tracking/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/iris_tracking/iris_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/iris_tracking/iris_tracking_gpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/iris_tracking:iris_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/iris_tracking/BUILD)
|
||||
|
||||
#### Video File Input
|
||||
|
||||
1. To build the application, run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/iris_tracking:iris_tracking_cpu_video_input
|
||||
```
|
||||
|
||||
2. To run the application, replace `<input video path>` and `<output video
|
||||
path>` in the command below with your own paths:
|
||||
|
||||
```
|
||||
bazel-bin/mediapipe/examples/desktop/iris_tracking/iris_tracking_cpu_video_input \
|
||||
--calculator_graph_config_file=mediapipe/graphs/iris_tracking/iris_tracking_cpu_video_input.pbtxt \
|
||||
--input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>
|
||||
```
|
||||
|
||||
#### Single-image Depth Estimation
|
||||
|
||||
1. To build the application, run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/iris_tracking:iris_depth_from_image_desktop
|
||||
```
|
||||
|
||||
2. To run the application, replace `<input image path>` and `<output image
|
||||
path>` in the command below with your own paths:
|
||||
|
||||
```bash
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop \
|
||||
--input_image_path=<input image path> --output_image_path=<output image path>
|
||||
```
|
||||
|
||||
### Web
|
||||
|
||||
Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
[MediaPipe Iris: Real-time Eye Tracking and Depth Estimation](https://ai.googleblog.com/2020/08/mediapipe-iris-real-time-iris-tracking.html)
|
||||
* Paper:
|
||||
[Real-time Pupil Tracking from Monocular Video for Digital Puppetry](https://arxiv.org/abs/2006.11341)
|
||||
([presentation](https://youtu.be/cIhXkiiapQI))
|
||||
* [Models and model cards](./models.md#iris)
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: KNIFT (Template-based Feature Matching)
|
||||
parent: Solutions
|
||||
nav_order: 8
|
||||
nav_order: 11
|
||||
---
|
||||
|
||||
# MediaPipe KNIFT
|
||||
@@ -139,7 +139,4 @@ to run regular TFLite inference.
|
||||
|
||||
* Google Developers Blog:
|
||||
[MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
|
||||
* [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
|
||||
* [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
|
||||
* [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
|
||||
* [Model card](https://mediapipe.page.link/knift-mc)
|
||||
* [Models and model cards](./models.md#knift)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: Dataset Preparation with MediaSequence
|
||||
parent: Solutions
|
||||
nav_order: 10
|
||||
nav_order: 13
|
||||
---
|
||||
|
||||
# Dataset Preparation with MediaSequence
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
---
|
||||
layout: default
|
||||
title: Models and Model Cards
|
||||
parent: Solutions
|
||||
nav_order: 30
|
||||
---
|
||||
|
||||
# MediaPipe Models and Model Cards
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
|
||||
|
||||
* Face detection model for front-facing/selfie camera:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
|
||||
[TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
|
||||
* Face detection model for back-facing camera:
|
||||
[TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
|
||||
* [Model card](https://mediapipe.page.link/blazeface-mc)
|
||||
|
||||
### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)
|
||||
|
||||
* Face landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
|
||||
* [Model card](https://mediapipe.page.link/facemesh-mc)
|
||||
|
||||
### [Iris](https://google.github.io/mediapipe/solutions/iris)
|
||||
|
||||
* Iris landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark.tflite)
|
||||
* [Model card](https://mediapipe.page.link/iris-mc)
|
||||
|
||||
### [Hands](https://google.github.io/mediapipe/solutions/hands)
|
||||
|
||||
* Palm detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
|
||||
* Hand landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
|
||||
* [Model card](https://mediapipe.page.link/handmc)
|
||||
|
||||
### [Pose](https://google.github.io/mediapipe/solutions/pose)
|
||||
|
||||
* Pose detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
|
||||
* Upper-body pose landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
|
||||
* [Model card](https://mediapipe.page.link/blazepose-mc)
|
||||
|
||||
### [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation)
|
||||
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
|
||||
* [Model card](https://mediapipe.page.link/hairsegmentation-mc)
|
||||
|
||||
### [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)
|
||||
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
|
||||
* [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
|
||||
* [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
|
||||
* [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
|
||||
|
||||
### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
|
||||
|
||||
* [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
|
||||
* [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
|
||||
* [TFLite model for cameras](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_camera.tflite)
|
||||
* [TFLite model for cups](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_cup.tflite)
|
||||
* [Single-stage TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers_1stage.tflite)
|
||||
* [Single-stage TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair_1stage.tflite)
|
||||
* [Model card](https://mediapipe.page.link/objectron-mc)
|
||||
|
||||
### [KNIFT](https://google.github.io/mediapipe/solutions/knift)
|
||||
|
||||
* [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
|
||||
* [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
|
||||
* [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
|
||||
* [Model card](https://mediapipe.page.link/knift-mc)
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: Object Detection
|
||||
parent: Solutions
|
||||
nav_order: 5
|
||||
nav_order: 7
|
||||
---
|
||||
|
||||
# MediaPipe Object Detection
|
||||
@@ -95,8 +95,8 @@ Please first see general instructions for
|
||||
|
||||
```
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt \
|
||||
--input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt \
|
||||
--input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>
|
||||
```
|
||||
|
||||
* With a TensorFlow Model
|
||||
@@ -131,8 +131,8 @@ Please first see general instructions for
|
||||
|
||||
```bash
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt \
|
||||
--input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt \
|
||||
--input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>
|
||||
```
|
||||
|
||||
### Coral
|
||||
@@ -144,7 +144,4 @@ to cross-compile and run MediaPipe examples on the
|
||||
|
||||
## Resources
|
||||
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
|
||||
* [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
|
||||
* [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
|
||||
* [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
|
||||
* [Models and model cards](./models.md#object_detection)
|
||||
|
||||
+100
-30
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: Objectron (3D Object Detection)
|
||||
parent: Solutions
|
||||
nav_order: 7
|
||||
nav_order: 10
|
||||
---
|
||||
|
||||
# MediaPipe Objectron
|
||||
@@ -15,13 +15,12 @@ nav_order: 7
|
||||
## Overview
|
||||
|
||||
MediaPipe Objectron is a mobile real-time 3D object detection solution for
|
||||
everyday objects. It detects objects in 2D images, and estimates their poses and
|
||||
sizes through a machine learning (ML) model, trained on a newly created 3D
|
||||
dataset.
|
||||
everyday objects. It detects objects in 2D images, and estimates their poses
|
||||
through a machine learning (ML) model, trained on a newly created 3D dataset.
|
||||
|
||||
 | 
|
||||
:--------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------:
|
||||
*Fig 1(a). Objectron for Shoes.* | *Fig 1(b). Objectron for Chairs.*
|
||||
 |  |  | 
|
||||
:--------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------: | :------------------------------------------------------------------------------------: | :------------------------------------------------------------------------------:
|
||||
*Fig 1(a). Objectron for Shoes.* | *Fig 1(b). Objectron for Chairs.* | *Fig 1(c). Objectron for Cameras.* | *Fig 1(d). Objectron for Cups.*
|
||||
|
||||
Object detection is an extensively studied computer vision problem, but most of
|
||||
the research has focused on
|
||||
@@ -85,15 +84,41 @@ able to increase the accuracy by about 10%.
|
||||
:-------------------------------------------------------------------------------------------: |
|
||||
*Fig 4. An example of AR synthetic data generation. The virtual white-brown cereal box is rendered into the real scene, next to the real blue book.* |
|
||||
|
||||
## ML Model for 3D Object Detection
|
||||
## ML Pipelines for 3D Object Detection
|
||||
|
||||
We built two ML pipelines to predict the 3D bounding box of an object from a
|
||||
single RGB image: one is a two-stage pipeline and the other is a single-stage
|
||||
pipeline. The two-stage pipeline is 3x faster than the single-stage pipeline
|
||||
with similar or better accuracy. The single stage pipeline is good at detecting
|
||||
multiple objects, whereas the two stage pipeline is good for a single dominant
|
||||
object.
|
||||
|
||||
### Two-stage Pipeline
|
||||
|
||||
Our two-stage pipeline is illustrated by the diagram in Fig 5. The first stage
|
||||
uses an object detector to find the 2D crop of the object. The second stage
|
||||
takes the image crop and estimates the 3D bounding box. At the same time, it
|
||||
also computes the 2D crop of the object for the next frame, such that the object
|
||||
detector does not need to run every frame.
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------------------: |
|
||||
*Fig 5. Network architecture and post-processing for two-stage 3D object detection.* |
|
||||
|
||||
We can use any 2D object detector for the first stage. In this solution, we use
|
||||
[TensorFlow Object Detection](https://github.com/tensorflow/models/tree/master/research/object_detection).
|
||||
The second stage 3D bounding box predictor we released runs 83FPS on Adreno 650
|
||||
mobile GPU.
|
||||
|
||||
### Single-stage Pipeline
|
||||
|
||||
 |
|
||||
:---------------------------------------------------------------------------------: |
|
||||
*Fig 5. Network architecture and post-processing for 3D object detection.* |
|
||||
*Fig 6. Network architecture and post-processing for single-stage 3D object detection.* |
|
||||
|
||||
We [built a single-stage model](https://arxiv.org/abs/2003.03522) to predict the
|
||||
pose and physical size of an object from a single RGB image. The model backbone
|
||||
has an encoder-decoder architecture, built upon
|
||||
Our [single-stage pipeline](https://arxiv.org/abs/2003.03522) is illustrated by
|
||||
the diagram in Fig 6, the model backbone has an encoder-decoder architecture,
|
||||
built upon
|
||||
[MobileNetv2](https://ai.googleblog.com/2018/04/mobilenetv2-next-generation-of-on.html).
|
||||
We employ a multi-task learning approach, jointly predicting an object's shape
|
||||
with detection and regression. The shape task predicts the object's shape
|
||||
@@ -114,9 +139,9 @@ size of the object. The model is light enough to run real-time on mobile devices
|
||||
|
||||
 |
|
||||
:-------------------------------------------------------------------------------------: |
|
||||
*Fig 6. Sample results of our network — (Left) original 2D image with estimated bounding boxes, (Middle) object detection by Gaussian distribution, (Right) predicted segmentation mask.* |
|
||||
*Fig 7. Sample results of our network — (Left) original 2D image with estimated bounding boxes, (Middle) object detection by Gaussian distribution, (Right) predicted segmentation mask.* |
|
||||
|
||||
## Detection and Tracking Pipeline
|
||||
#### Detection and Tracking
|
||||
|
||||
When the model is applied to every frame captured by the mobile device, it can
|
||||
suffer from jitter due to the ambiguity of the 3D bounding box estimated in each
|
||||
@@ -130,7 +155,7 @@ temporally consistent, reducing the jitter.
|
||||
|
||||
The Objectron 3D object detection and tracking pipeline is implemented as a
|
||||
MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt),
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt),
|
||||
which internally uses a
|
||||
[detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/subgraphs/objectron_detection_gpu.pbtxt)
|
||||
and a
|
||||
@@ -147,6 +172,12 @@ new detection becomes available from the detection subgraph, the tracking
|
||||
subgraph is also responsible for consolidation between the detection and
|
||||
tracking results, based on the area of overlap.
|
||||
|
||||
## Objectron Dataset
|
||||
|
||||
We also released our [Objectron dataset](http://objectron.dev), with which we
|
||||
trained our 3D object detection models. The technical details of the Objectron
|
||||
dataset, including usage and tutorials, are available on the dataset website.
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
@@ -158,32 +189,72 @@ Note: To visualize a graph, copy the graph and paste it into
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
### Objectron for Shoes
|
||||
### Two-stage Objectron
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1S0K4hbWt3o31FfQ4QU3Rz7IHrvOUMx1d)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD)
|
||||
* iOS target: Not available
|
||||
[`mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
|
||||
|
||||
### Objectron for Chairs
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/chair_classic_occlusion_tracking.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1MM8K-13bXLCVS1EHQ-KgkVyEahEPrKej)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD)
|
||||
and add `--define chair=true` to the build command, i.e.,
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD).
|
||||
|
||||
Build for **shoes** (default) with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1ANW9WDOCb8QO1r8gDC03A4UgrPkICdPP/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Build for **chairs** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1lcUv1TBnv_SxnKSQwdOqbdLa9mkaTJHy/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define chair=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Build for **cups** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1bf77KDkowwrduleiC9B1M1XnEhjnOQbX/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define cup=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Build for **cameras** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1GM7lPO-s5URVxIzQur1bLsionEJs3yIl/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define camera=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
### Single-stage Objectron
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
|
||||
|
||||
* Android target:
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD).
|
||||
|
||||
Build with **single-stage** model for **shoes** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1MvaEg4dkvKN8jAU1Z2GtudyXi1rQHYsE/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define shoe_1stage=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Build with **single-stage** model for **chairs** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1GJL4z3jr-wD1jMHGd4NBfOG-Yoq5t167/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define chair_1stage=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
[Announcing the Objectron Dataset](https://mediapipe.page.link/objectron_dataset_ai_blog)
|
||||
* Google AI Blog:
|
||||
[Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
|
||||
* Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
|
||||
@@ -191,5 +262,4 @@ to visualize its associated subgraphs, please see
|
||||
* Paper:
|
||||
[Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
|
||||
([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
|
||||
* [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
|
||||
* [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
|
||||
* [Models and model cards](./models.md#objectron)
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
---
|
||||
layout: default
|
||||
title: Pose
|
||||
parent: Solutions
|
||||
nav_order: 5
|
||||
---
|
||||
|
||||
# MediaPipe Pose
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Human pose estimation from video plays a critical role in various applications
|
||||
such as quantifying physical exercises, sign language recognition, and full-body
|
||||
gesture control. For example, it can form the basis for yoga, dance, and fitness
|
||||
applications. It can also enable the overlay of digital content and information
|
||||
on top of the physical world in augmented reality.
|
||||
|
||||
MediaPipe Pose is a ML solution for high-fidelity upper-body pose tracking,
|
||||
inferring 25 2D upper-body landmarks from RGB video frames utilizing our
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
research. Current state-of-the-art approaches rely primarily on powerful desktop
|
||||
environments for inference, whereas our method achieves real-time performance on
|
||||
most modern [mobile phones](#mobile), [desktops/laptops](#desktop), in
|
||||
[python](#python) and even on the [web](#web). A variant of MediaPipe Pose that
|
||||
performs full-body pose tracking on mobile phones will be included in an
|
||||
upcoming release of
|
||||
[ML Kit](https://developers.google.com/ml-kit/early-access/pose-detection).
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------: |
|
||||
*Fig 1. Example of MediaPipe Pose for upper-body pose tracking.* |
|
||||
|
||||
## ML Pipeline
|
||||
|
||||
The solution utilizes a two-step detector-tracker ML pipeline, proven to be
|
||||
effective in our [MediaPipe Hands](./hands.md) and
|
||||
[MediaPipe Face Mesh](./face_mesh.md) solutions. Using a detector, the pipeline
|
||||
first locates the pose region-of-interest (ROI) within the frame. The tracker
|
||||
subsequently predicts the pose landmarks within the ROI using the ROI-cropped
|
||||
frame as input. Note that for video use cases the detector is invoked only as
|
||||
needed, i.e., for the very first frame and when the tracker could no longer
|
||||
identify body pose presence in the previous frame. For other frames the pipeline
|
||||
simply derives the ROI from the previous frame’s pose landmarks.
|
||||
|
||||
The pipeline is implemented as a MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt)
|
||||
that uses a
|
||||
[pose landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_gpu.pbtxt)
|
||||
from the
|
||||
[pose landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark)
|
||||
and renders using a dedicated
|
||||
[upper-body pose renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/subgraphs/upper_body_pose_renderer_gpu.pbtxt).
|
||||
The
|
||||
[pose landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_gpu.pbtxt)
|
||||
internally uses a
|
||||
[pose detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection_gpu.pbtxt)
|
||||
from the
|
||||
[pose detection module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection).
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
## Models
|
||||
|
||||
### Pose Detection Model (BlazePose Detector)
|
||||
|
||||
The detector is inspired by our own lightweight
|
||||
[BlazeFace](https://arxiv.org/abs/1907.05047) model, used in
|
||||
[MediaPipe Face Detection](./face_detection.md), as a proxy for a person
|
||||
detector. It explicitly predicts two additional virtual keypoints that firmly
|
||||
describe the human body center, rotation and scale as a circle. Inspired by
|
||||
[Leonardo’s Vitruvian man](https://en.wikipedia.org/wiki/Vitruvian_Man), we
|
||||
predict the midpoint of a person's hips, the radius of a circle circumscribing
|
||||
the whole person, and the incline angle of the line connecting the shoulder and
|
||||
hip midpoints.
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------------------------------: |
|
||||
*Fig 2. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
|
||||
|
||||
### Pose Landmark Model (BlazePose Tracker)
|
||||
|
||||
The landmark model currently included in MediaPipe Pose predicts the location of
|
||||
25 upper-body landmarks (see figure below), each with `(x, y, z, visibility)`.
|
||||
Note that the `z` value should be discarded as the model is currently not fully
|
||||
trained to predict depth, but this is something we have on the roadmap. The
|
||||
model shares the same architecture as the full-body version that predicts 33
|
||||
landmarks, described in more detail in the
|
||||
[BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
and in this [paper](https://arxiv.org/abs/2006.10204).
|
||||
|
||||
 |
|
||||
:------------------------------------------------------------------------------------------------: |
|
||||
*Fig 3. 25 upper-body pose landmarks.* |
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android),
|
||||
[iOS](../getting_started/building_examples.md#ios),
|
||||
[desktop](../getting_started/building_examples.md#desktop) and
|
||||
[Python](../getting_started/building_examples.md#python) on how to build
|
||||
MediaPipe examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
### Mobile
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1uKc6T7KSuA0Mlq2URi5YookHu0U3yoh_/view?usp=sharing)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/upperbodyposetrackinggpu:upperbodyposetrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/upperbodyposetrackinggpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp`](http:/mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD)
|
||||
|
||||
### Desktop
|
||||
|
||||
Please first see general instructions for
|
||||
[desktop](../getting_started/building_examples.md#desktop) on how to build
|
||||
MediaPipe examples.
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_cpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_cpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
|
||||
|
||||
### Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described below and in this
|
||||
[colab](https://mediapipe.page.link/pose_py_colab). If you do need to build the
|
||||
Python package from source, see
|
||||
[additional instructions](../getting_started/building_examples.md#python).
|
||||
|
||||
Activate a Python virtual environment:
|
||||
|
||||
```bash
|
||||
$ python3 -m venv mp_env && source mp_env/bin/activate
|
||||
```
|
||||
|
||||
Install MediaPipe Python package:
|
||||
|
||||
```bash
|
||||
(mp_env)$ pip install mediapipe
|
||||
```
|
||||
|
||||
Run the following Python code:
|
||||
|
||||
<!-- Do not change the example code below directly. Change the corresponding example in mediapipe/python/solutions/pose.py and copy it over. -->
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_pose = mp.solutions.pose
|
||||
|
||||
# For static images:
|
||||
pose = mp_pose.Pose(
|
||||
static_image_mode=True, min_detection_confidence=0.5)
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Print and draw pose landmarks on the image.
|
||||
print(
|
||||
'nose landmark:',
|
||||
results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE])
|
||||
annotated_image = image.copy()
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', image)
|
||||
pose.close()
|
||||
|
||||
# For webcam input:
|
||||
pose = mp_pose.Pose(
|
||||
min_detection_confidence=0.5, min_tracking_confidence=0.5)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = pose.process(image)
|
||||
|
||||
# Draw the pose annotation on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Pose', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
pose.close()
|
||||
cap.release()
|
||||
```
|
||||
|
||||
Tip: Use command `deactivate` to exit the Python virtual environment.
|
||||
|
||||
### Web
|
||||
|
||||
Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
[BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
* Paper:
|
||||
[BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
|
||||
([presentation](https://youtu.be/YPpUOTRn5tA))
|
||||
* [Models and model cards](./models.md#pose)
|
||||
+21
-14
@@ -14,18 +14,25 @@ has_toc: false
|
||||
---
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to ../index.md. -->
|
||||
<!-- Whenever this table is updated, paste a copy to ../external_index.md. -->
|
||||
|
||||
[]() | Android | iOS | Desktop | Web | Coral
|
||||
:---------------------------------------------------------------------------- | :-----: | :-: | :-----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | |
|
||||
[]() | Android | iOS | Desktop | Python | Web | Coral
|
||||
:---------------------------------------------------------------------------------------- | :-----: | :-: | :-----: | :----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: YouTube-8M Feature Extraction and Model Inference
|
||||
parent: Solutions
|
||||
nav_order: 11
|
||||
nav_order: 14
|
||||
---
|
||||
|
||||
# YouTube-8M Feature Extraction and Model Inference
|
||||
|
||||
@@ -26,9 +26,10 @@ To enable tracing and profiling of a mediapipe graph:
|
||||
1. The profiling library must be linked to the framework.
|
||||
2. Tracing and profiling must be enabled in the graph configuration.
|
||||
|
||||
The profiling library is linked to the framework by default. If needed,
|
||||
the profiling library can be omitted from the framework using the bazel
|
||||
command line option: `--define MEDIAPIPE_PROFILING=0`.
|
||||
The profiling library is linked to the framework by default for Desktop.
|
||||
If needed, it can be omitted from the framework using the bazel command line
|
||||
option: `--define MEDIAPIPE_PROFILING=0`. For other platforms, you can use the
|
||||
bazel command line option `--define MEDIAPIPE_PROFILING=1` to link it.
|
||||
|
||||
To enable tracing and profiling, the `CalculatorGraphConfig` (in
|
||||
[calculator.proto](https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator.proto))
|
||||
@@ -38,6 +39,7 @@ is a simple setup that turns on tracing and keeps 100 seconds of timing events:
|
||||
```
|
||||
profiler_config {
|
||||
trace_enabled: true
|
||||
enable_profiler: true
|
||||
trace_log_interval_count: 200
|
||||
}
|
||||
```
|
||||
@@ -71,6 +73,9 @@ MediaPipe will emit data into a pre-specified directory:
|
||||
|
||||
You can open the Download Container. Logs will be located in `application
|
||||
container/.xcappdata/AppData/Documents/`
|
||||
If XCode shows empty content for the downloaded container file, you can
|
||||
right click and select 'Show Package Contents' in Finder. Logs
|
||||
will be located in 'AppData/Documents/'
|
||||
|
||||
|
||||
|
||||
@@ -144,10 +149,14 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
```bash
|
||||
profiler_config {
|
||||
trace_enabled: true
|
||||
trace_log_path: "/sdcard/profiles"
|
||||
enable_profiler: true
|
||||
trace_log_path: "/sdcard/profiles/"
|
||||
}
|
||||
```
|
||||
|
||||
Note: The forward slash at the end of the `trace_log_path` is necessary for
|
||||
indicating that `profiles` is a directory (that *should* exist).
|
||||
|
||||
* Download the trace files from the device.
|
||||
|
||||
```bash
|
||||
@@ -294,7 +303,7 @@ trace_log_margin_usec
|
||||
in trace log output. This margin allows time for events to be appended to
|
||||
the TraceBuffer.
|
||||
|
||||
trace_log_duration_events
|
||||
trace_log_instant_events
|
||||
: False specifies an event for each calculator invocation. True specifies a
|
||||
separate event for each start and finish time.
|
||||
|
||||
|
||||
@@ -3,27 +3,34 @@
|
||||
"/BUILD",
|
||||
"mediapipe/BUILD",
|
||||
"mediapipe/objc/BUILD",
|
||||
"mediapipe/examples/ios/BUILD",
|
||||
"mediapipe/examples/ios/edgedetectiongpu/BUILD",
|
||||
"mediapipe/framework/BUILD",
|
||||
"mediapipe/gpu/BUILD",
|
||||
"mediapipe/objc/testing/app/BUILD",
|
||||
"mediapipe/examples/ios/common/BUILD",
|
||||
"mediapipe/examples/ios/helloworld/BUILD",
|
||||
"mediapipe/examples/ios/facedetectioncpu/BUILD",
|
||||
"mediapipe/examples/ios/facedetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/faceeffect/BUILD",
|
||||
"mediapipe/examples/ios/facemeshgpu/BUILD",
|
||||
"mediapipe/examples/ios/handdetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/handtrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/iristrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/objectdetectioncpu/BUILD",
|
||||
"mediapipe/examples/ios/objectdetectiongpu/BUILD"
|
||||
"mediapipe/examples/ios/objectdetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD"
|
||||
],
|
||||
"buildTargets" : [
|
||||
"//mediapipe/examples/ios/edgedetectiongpu:EdgeDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/helloworld:HelloWorldApp",
|
||||
"//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp",
|
||||
"//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/faceeffect:FaceEffectApp",
|
||||
"//mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp",
|
||||
"//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/iristrackinggpu:IrisTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp",
|
||||
"//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp",
|
||||
"//mediapipe/objc:mediapipe_framework_ios"
|
||||
],
|
||||
"optionSet" : {
|
||||
@@ -78,22 +85,19 @@
|
||||
"mediapipe/calculators/util",
|
||||
"mediapipe/examples",
|
||||
"mediapipe/examples/ios",
|
||||
"mediapipe/examples/ios/edgedetectiongpu",
|
||||
"mediapipe/examples/ios/edgedetectiongpu/Base.lproj",
|
||||
"mediapipe/examples/ios/common",
|
||||
"mediapipe/examples/ios/common/Base.lproj",
|
||||
"mediapipe/examples/ios/helloworld",
|
||||
"mediapipe/examples/ios/facedetectioncpu",
|
||||
"mediapipe/examples/ios/facedetectioncpu/Base.lproj",
|
||||
"mediapipe/examples/ios/facedetectiongpu",
|
||||
"mediapipe/examples/ios/facedetectiongpu/Base.lproj",
|
||||
"mediapipe/examples/ios/faceeffect",
|
||||
"mediapipe/examples/ios/faceeffect/Base.lproj",
|
||||
"mediapipe/examples/ios/handdetectiongpu",
|
||||
"mediapipe/examples/ios/handdetectiongpu/Base.lproj",
|
||||
"mediapipe/examples/ios/handtrackinggpu",
|
||||
"mediapipe/examples/ios/handtrackinggpu/Base.lproj",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu/Base.lproj",
|
||||
"mediapipe/examples/ios/iristrackinggpu",
|
||||
"mediapipe/examples/ios/objectdetectioncpu",
|
||||
"mediapipe/examples/ios/objectdetectioncpu/Base.lproj",
|
||||
"mediapipe/examples/ios/objectdetectiongpu",
|
||||
"mediapipe/examples/ios/objectdetectiongpu/Base.lproj",
|
||||
"mediapipe/examples/ios/upperbodyposetrackinggpu",
|
||||
"mediapipe/framework",
|
||||
"mediapipe/framework/deps",
|
||||
"mediapipe/framework/formats",
|
||||
@@ -107,9 +111,12 @@
|
||||
"mediapipe/graphs",
|
||||
"mediapipe/graphs/edge_detection",
|
||||
"mediapipe/graphs/face_detection",
|
||||
"mediapipe/graphs/face_geometry",
|
||||
"mediapipe/graphs/hand_tracking",
|
||||
"mediapipe/graphs/object_detection",
|
||||
"mediapipe/graphs/pose_tracking",
|
||||
"mediapipe/models",
|
||||
"mediapipe/modules",
|
||||
"mediapipe/objc",
|
||||
"mediapipe/util",
|
||||
"mediapipe/util/android",
|
||||
|
||||
@@ -11,15 +11,16 @@
|
||||
"mediapipe",
|
||||
"mediapipe/objc",
|
||||
"mediapipe/examples/ios",
|
||||
"mediapipe/examples/ios/edgedetectiongpu",
|
||||
"mediapipe/examples/ios/facedetectioncpu",
|
||||
"mediapipe/examples/ios/facedetectiongpu",
|
||||
"mediapipe/examples/ios/faceeffect",
|
||||
"mediapipe/examples/ios/facemeshgpu",
|
||||
"mediapipe/examples/ios/handdetectiongpu",
|
||||
"mediapipe/examples/ios/handtrackinggpu",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu",
|
||||
"mediapipe/examples/ios/iristrackinggpu",
|
||||
"mediapipe/examples/ios/objectdetectioncpu",
|
||||
"mediapipe/examples/ios/objectdetectiongpu"
|
||||
"mediapipe/examples/ios/objectdetectiongpu",
|
||||
"mediapipe/examples/ios/upperbodyposetrackinggpu"
|
||||
],
|
||||
"projectName" : "Mediapipe",
|
||||
"workspaceRoot" : "../.."
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Copyright 2019 The MediaPipe Authors.
|
||||
"""Copyright 2019 - 2020 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
filegroup(
|
||||
name = "test_audios",
|
||||
|
||||
+132
-109
@@ -13,181 +13,121 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "concatenate_vector_calculator_proto",
|
||||
srcs = ["concatenate_vector_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "dequantize_byte_array_calculator_proto",
|
||||
srcs = ["dequantize_byte_array_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "packet_cloner_calculator_proto",
|
||||
srcs = ["packet_cloner_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "packet_resampler_calculator_proto",
|
||||
srcs = ["packet_resampler_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "packet_thinner_calculator_proto",
|
||||
srcs = ["packet_thinner_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "split_vector_calculator_proto",
|
||||
srcs = ["split_vector_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "quantize_float_vector_calculator_proto",
|
||||
srcs = ["quantize_float_vector_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "sequence_shift_calculator_proto",
|
||||
srcs = ["sequence_shift_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "gate_calculator_proto",
|
||||
srcs = ["gate_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "constant_side_packet_calculator_proto",
|
||||
srcs = ["constant_side_packet_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/framework/formats:classification_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "clip_vector_size_calculator_proto",
|
||||
srcs = ["clip_vector_size_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "packet_cloner_calculator_cc_proto",
|
||||
srcs = ["packet_cloner_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":packet_cloner_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "packet_resampler_calculator_cc_proto",
|
||||
srcs = ["packet_resampler_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":packet_resampler_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "packet_thinner_calculator_cc_proto",
|
||||
srcs = ["packet_thinner_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":packet_thinner_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "split_vector_calculator_cc_proto",
|
||||
srcs = ["split_vector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":split_vector_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "concatenate_vector_calculator_cc_proto",
|
||||
srcs = ["concatenate_vector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":concatenate_vector_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "clip_vector_size_calculator_cc_proto",
|
||||
srcs = ["clip_vector_size_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":clip_vector_size_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "dequantize_byte_array_calculator_cc_proto",
|
||||
srcs = ["dequantize_byte_array_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":dequantize_byte_array_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "quantize_float_vector_calculator_cc_proto",
|
||||
srcs = ["quantize_float_vector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":quantize_float_vector_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "sequence_shift_calculator_cc_proto",
|
||||
srcs = ["sequence_shift_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":sequence_shift_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "gate_calculator_cc_proto",
|
||||
srcs = ["gate_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":gate_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "constant_side_packet_calculator_cc_proto",
|
||||
srcs = ["constant_side_packet_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":constant_side_packet_calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
@@ -286,11 +226,23 @@ cc_library(
|
||||
name = "concatenate_vector_calculator",
|
||||
srcs = ["concatenate_vector_calculator.cc"],
|
||||
hdrs = ["concatenate_vector_calculator.h"],
|
||||
copts = select({
|
||||
# Needed for "//mediapipe/framework/formats:tensor" compatibility on Apple
|
||||
# platforms for Metal pulled in via the tensor.h header.
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":concatenate_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
@@ -316,6 +268,37 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "concatenate_normalized_landmark_list_calculator",
|
||||
srcs = ["concatenate_normalized_landmark_list_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":concatenate_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "concatenate_normalized_landmark_list_calculator_test",
|
||||
srcs = ["concatenate_normalized_landmark_list_calculator_test.cc"],
|
||||
deps = [
|
||||
":concatenate_normalized_landmark_list_calculator",
|
||||
":concatenate_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "concatenate_vector_calculator_test",
|
||||
srcs = ["concatenate_vector_calculator_test.cc"],
|
||||
@@ -575,6 +558,35 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "packet_presence_calculator",
|
||||
srcs = ["packet_presence_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "packet_presence_calculator_test",
|
||||
srcs = ["packet_presence_calculator_test.cc"],
|
||||
deps = [
|
||||
":gate_calculator",
|
||||
":packet_presence_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:sink",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "previous_loopback_calculator",
|
||||
srcs = ["previous_loopback_calculator.cc"],
|
||||
@@ -790,14 +802,23 @@ cc_library(
|
||||
name = "split_vector_calculator",
|
||||
srcs = ["split_vector_calculator.cc"],
|
||||
hdrs = ["split_vector_calculator.h"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":split_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:resource_util",
|
||||
@@ -1059,6 +1080,8 @@ cc_library(
|
||||
":constant_side_packet_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message ClipVectorSizeCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional ClipVectorSizeCalculatorOptions ext = 274674998;
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_ // NOLINT
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_ // NOLINT
|
||||
|
||||
#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Concatenates several NormalizedLandmarkList protos following stream index
|
||||
// order. This class assumes that every input stream contains a
|
||||
// NormalizedLandmarkList proto object.
|
||||
class ConcatenateNormalizedLandmarkListCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().NumEntries() != 0);
|
||||
RET_CHECK(cc->Outputs().NumEntries() == 1);
|
||||
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
cc->Inputs().Index(i).Set<NormalizedLandmarkList>();
|
||||
}
|
||||
|
||||
cc->Outputs().Index(0).Set<NormalizedLandmarkList>();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
only_emit_if_all_present_ =
|
||||
cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
|
||||
.only_emit_if_all_present();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (only_emit_if_all_present_) {
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (cc->Inputs().Index(i).IsEmpty()) return ::mediapipe::OkStatus();
|
||||
}
|
||||
}
|
||||
|
||||
NormalizedLandmarkList output;
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (cc->Inputs().Index(i).IsEmpty()) continue;
|
||||
const NormalizedLandmarkList& input =
|
||||
cc->Inputs().Index(i).Get<NormalizedLandmarkList>();
|
||||
for (int j = 0; j < input.landmark_size(); ++j) {
|
||||
const NormalizedLandmark& input_landmark = input.landmark(j);
|
||||
*output.add_landmark() = input_landmark;
|
||||
}
|
||||
}
|
||||
cc->Outputs().Index(0).AddPacket(
|
||||
MakePacket<NormalizedLandmarkList>(output).At(cc->InputTimestamp()));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
bool only_emit_if_all_present_;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(ConcatenateNormalizedLandmarkListCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
// NOLINTNEXTLINE
|
||||
#endif // MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_
|
||||
@@ -0,0 +1,184 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr float kLocationValue = 3;
|
||||
|
||||
NormalizedLandmarkList GenerateLandmarks(int landmarks_size,
|
||||
int value_multiplier) {
|
||||
NormalizedLandmarkList landmarks;
|
||||
for (int i = 0; i < landmarks_size; ++i) {
|
||||
NormalizedLandmark* landmark = landmarks.add_landmark();
|
||||
landmark->set_x(value_multiplier * kLocationValue);
|
||||
landmark->set_y(value_multiplier * kLocationValue);
|
||||
landmark->set_z(value_multiplier * kLocationValue);
|
||||
}
|
||||
return landmarks;
|
||||
}
|
||||
|
||||
void ValidateCombinedLandmarks(
|
||||
const std::vector<NormalizedLandmarkList>& inputs,
|
||||
const NormalizedLandmarkList& result) {
|
||||
int element_id = 0;
|
||||
int expected_size = 0;
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
const NormalizedLandmarkList& landmarks_i = inputs[i];
|
||||
expected_size += landmarks_i.landmark_size();
|
||||
for (int j = 0; j < landmarks_i.landmark_size(); ++j) {
|
||||
const NormalizedLandmark& expected = landmarks_i.landmark(j);
|
||||
const NormalizedLandmark& got = result.landmark(element_id);
|
||||
EXPECT_FLOAT_EQ(expected.x(), got.x());
|
||||
EXPECT_FLOAT_EQ(expected.y(), got.y());
|
||||
EXPECT_FLOAT_EQ(expected.z(), got.z());
|
||||
++element_id;
|
||||
}
|
||||
}
|
||||
EXPECT_EQ(expected_size, result.landmark_size());
|
||||
}
|
||||
|
||||
void AddInputLandmarkLists(
|
||||
const std::vector<NormalizedLandmarkList>& input_landmarks_vec,
|
||||
int64 timestamp, CalculatorRunner* runner) {
|
||||
for (int i = 0; i < input_landmarks_vec.size(); ++i) {
|
||||
runner->MutableInputs()->Index(i).packets.push_back(
|
||||
MakePacket<NormalizedLandmarkList>(input_landmarks_vec[i])
|
||||
.At(Timestamp(timestamp)));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ConcatenateNormalizedLandmarkListCalculatorTest, EmptyVectorInputs) {
|
||||
CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
NormalizedLandmarkList empty_list;
|
||||
std::vector<NormalizedLandmarkList> inputs = {empty_list, empty_list,
|
||||
empty_list};
|
||||
AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(0, outputs[0].Get<NormalizedLandmarkList>().landmark_size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
}
|
||||
|
||||
TEST(ConcatenateNormalizedLandmarkListCalculatorTest, OneTimestamp) {
|
||||
CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
NormalizedLandmarkList input_0 =
|
||||
GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0);
|
||||
NormalizedLandmarkList input_1 =
|
||||
GenerateLandmarks(/*landmarks_size=*/1, /*value_multiplier=*/1);
|
||||
NormalizedLandmarkList input_2 =
|
||||
GenerateLandmarks(/*landmarks_size=*/2, /*value_multiplier=*/2);
|
||||
std::vector<NormalizedLandmarkList> inputs = {input_0, input_1, input_2};
|
||||
AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
const NormalizedLandmarkList& result =
|
||||
outputs[0].Get<NormalizedLandmarkList>();
|
||||
ValidateCombinedLandmarks(inputs, result);
|
||||
}
|
||||
|
||||
TEST(ConcatenateNormalizedLandmarkListCalculatorTest,
|
||||
TwoInputsAtTwoTimestamps) {
|
||||
CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
NormalizedLandmarkList input_0 =
|
||||
GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0);
|
||||
NormalizedLandmarkList input_1 =
|
||||
GenerateLandmarks(/*landmarks_size=*/1, /*value_multiplier=*/1);
|
||||
NormalizedLandmarkList input_2 =
|
||||
GenerateLandmarks(/*landmarks_size=*/2, /*value_multiplier=*/2);
|
||||
std::vector<NormalizedLandmarkList> inputs = {input_0, input_1, input_2};
|
||||
{ AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner); }
|
||||
{ AddInputLandmarkLists(inputs, /*timestamp=*/2, &runner); }
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(2, outputs.size());
|
||||
{
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
const NormalizedLandmarkList& result =
|
||||
outputs[0].Get<NormalizedLandmarkList>();
|
||||
ValidateCombinedLandmarks(inputs, result);
|
||||
}
|
||||
{
|
||||
EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
|
||||
const NormalizedLandmarkList& result =
|
||||
outputs[1].Get<NormalizedLandmarkList>();
|
||||
ValidateCombinedLandmarks(inputs, result);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ConcatenateNormalizedLandmarkListCalculatorTest,
|
||||
OneEmptyStreamStillOutput) {
|
||||
CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/2,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
NormalizedLandmarkList input_0 =
|
||||
GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0);
|
||||
std::vector<NormalizedLandmarkList> inputs = {input_0};
|
||||
AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
const NormalizedLandmarkList& result =
|
||||
outputs[0].Get<NormalizedLandmarkList>();
|
||||
ValidateCombinedLandmarks(inputs, result);
|
||||
}
|
||||
|
||||
TEST(ConcatenateNormalizedLandmarkListCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
|
||||
/*options_string=*/
|
||||
"[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
|
||||
"{only_emit_if_all_present: true}",
|
||||
/*num_inputs=*/2,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
NormalizedLandmarkList input_0 =
|
||||
GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0);
|
||||
std::vector<NormalizedLandmarkList> inputs = {input_0};
|
||||
AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -16,7 +16,10 @@
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
@@ -45,6 +48,12 @@ REGISTER_CALCULATOR(ConcatenateFloatVectorCalculator);
|
||||
typedef ConcatenateVectorCalculator<int32> ConcatenateInt32VectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<uint64> ConcatenateUInt64VectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateUInt64VectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateBoolVectorCalculator);
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
|
||||
@@ -56,10 +65,21 @@ typedef ConcatenateVectorCalculator<TfLiteTensor>
|
||||
ConcatenateTfLiteTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<Tensor> ConcatenateTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateTensorVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
|
||||
ConcatenateLandmarkVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmarkList>
|
||||
ConcatenateLandmarListVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateLandmarListVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<mediapipe::ClassificationList>
|
||||
ConcatenateClassificationListVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateClassificationListVectorCalculator);
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
|
||||
ConcatenateGlBufferVectorCalculator;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
|
||||
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
@@ -26,10 +27,10 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Concatenates several std::vector<T> following stream index order. This class
|
||||
// assumes that every input stream contains the vector<T> type. To use this
|
||||
// class for a particular type T, regisiter a calculator using
|
||||
// ConcatenateVectorCalculator<T>.
|
||||
// Concatenates several objects of type T or std::vector<T> following stream
|
||||
// index order. This class assumes that every input stream contains either T or
|
||||
// vector<T> type. To use this class for a particular type T, regisiter a
|
||||
// calculator using ConcatenateVectorCalculator<T>.
|
||||
template <typename T>
|
||||
class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
public:
|
||||
@@ -38,7 +39,8 @@ class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
RET_CHECK(cc->Outputs().NumEntries() == 1);
|
||||
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
cc->Inputs().Index(i).Set<std::vector<T>>();
|
||||
// Actual type T or vector<T> will be validated in Process().
|
||||
cc->Inputs().Index(i).SetAny();
|
||||
}
|
||||
|
||||
cc->Outputs().Index(0).Set<std::vector<T>>();
|
||||
@@ -69,9 +71,19 @@ class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
CalculatorContext* cc) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (cc->Inputs().Index(i).IsEmpty()) continue;
|
||||
const std::vector<U>& input = cc->Inputs().Index(i).Get<std::vector<U>>();
|
||||
output->insert(output->end(), input.begin(), input.end());
|
||||
auto& input = cc->Inputs().Index(i);
|
||||
|
||||
if (input.IsEmpty()) continue;
|
||||
|
||||
if (input.Value().ValidateAsType<U>().ok()) {
|
||||
const U& value = input.Get<U>();
|
||||
output->push_back(value);
|
||||
} else if (input.Value().ValidateAsType<std::vector<U>>().ok()) {
|
||||
const std::vector<U>& value = input.Get<std::vector<U>>();
|
||||
output->insert(output->end(), value.begin(), value.end());
|
||||
} else {
|
||||
return ::mediapipe::InvalidArgumentError("Invalid input stream type.");
|
||||
}
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
@@ -88,17 +100,32 @@ class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
CalculatorContext* cc) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (cc->Inputs().Index(i).IsEmpty()) continue;
|
||||
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
|
||||
cc->Inputs().Index(i).Value().Consume<std::vector<U>>();
|
||||
if (input_status.ok()) {
|
||||
std::unique_ptr<std::vector<U>> input_vector =
|
||||
std::move(input_status).ValueOrDie();
|
||||
output->insert(output->end(),
|
||||
std::make_move_iterator(input_vector->begin()),
|
||||
std::make_move_iterator(input_vector->end()));
|
||||
auto& input = cc->Inputs().Index(i);
|
||||
|
||||
if (input.IsEmpty()) continue;
|
||||
|
||||
if (input.Value().ValidateAsType<U>().ok()) {
|
||||
::mediapipe::StatusOr<std::unique_ptr<U>> value_status =
|
||||
input.Value().Consume<U>();
|
||||
if (value_status.ok()) {
|
||||
std::unique_ptr<U> value = std::move(value_status).ValueOrDie();
|
||||
output->push_back(std::move(*value));
|
||||
} else {
|
||||
return value_status.status();
|
||||
}
|
||||
} else if (input.Value().ValidateAsType<std::vector<U>>().ok()) {
|
||||
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> value_status =
|
||||
input.Value().Consume<std::vector<U>>();
|
||||
if (value_status.ok()) {
|
||||
std::unique_ptr<std::vector<U>> value =
|
||||
std::move(value_status).ValueOrDie();
|
||||
output->insert(output->end(), std::make_move_iterator(value->begin()),
|
||||
std::make_move_iterator(value->end()));
|
||||
} else {
|
||||
return value_status.status();
|
||||
}
|
||||
} else {
|
||||
return input_status.status();
|
||||
return ::mediapipe::InvalidArgumentError("Invalid input stream type.");
|
||||
}
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
@@ -109,7 +136,7 @@ class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
::mediapipe::Status ConsumeAndConcatenateVectors(std::false_type,
|
||||
CalculatorContext* cc) {
|
||||
return ::mediapipe::InternalError(
|
||||
"Cannot copy or move input vectors to concatenate them");
|
||||
"Cannot copy or move inputs to concatenate them");
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message ConcatenateVectorCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional ConcatenateVectorCalculatorOptions ext = 259397839;
|
||||
|
||||
@@ -30,11 +30,29 @@ namespace mediapipe {
|
||||
typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
|
||||
REGISTER_CALCULATOR(TestConcatenateIntVectorCalculator);
|
||||
|
||||
void AddInputVector(int index, const std::vector<int>& input, int64 timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
runner->MutableInputs()->Index(index).packets.push_back(
|
||||
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
|
||||
}
|
||||
|
||||
void AddInputVectors(const std::vector<std::vector<int>>& inputs,
|
||||
int64 timestamp, CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
runner->MutableInputs()->Index(i).packets.push_back(
|
||||
MakePacket<std::vector<int>>(inputs[i]).At(Timestamp(timestamp)));
|
||||
AddInputVector(i, inputs[i], timestamp, runner);
|
||||
}
|
||||
}
|
||||
|
||||
void AddInputItem(int index, int input, int64 timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
runner->MutableInputs()->Index(index).packets.push_back(
|
||||
MakePacket<int>(input).At(Timestamp(timestamp)));
|
||||
}
|
||||
|
||||
void AddInputItems(const std::vector<int>& inputs, int64 timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
AddInputItem(i, inputs[i], timestamp, runner);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,6 +149,135 @@ TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, ItemsOneTimestamp) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<int> inputs = {1, 2, 3};
|
||||
AddInputItems(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, ItemsTwoInputsAtTwoTimestamps) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
{
|
||||
std::vector<int> inputs = {1, 2, 3};
|
||||
AddInputItems(inputs, /*timestamp=*/1, &runner);
|
||||
}
|
||||
{
|
||||
std::vector<int> inputs = {4, 5, 6};
|
||||
AddInputItems(inputs, /*timestamp=*/2, &runner);
|
||||
}
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(2, outputs.size());
|
||||
{
|
||||
EXPECT_EQ(3, outputs[0].Get<std::vector<int>>().size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
{
|
||||
EXPECT_EQ(3, outputs[1].Get<std::vector<int>>().size());
|
||||
EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
|
||||
std::vector<int> expected_vector = {4, 5, 6};
|
||||
EXPECT_EQ(expected_vector, outputs[1].Get<std::vector<int>>());
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, ItemsOneEmptyStreamStillOutput) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
// No third input item.
|
||||
std::vector<int> inputs = {1, 2};
|
||||
AddInputItems(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, ItemsOneEmptyStreamNoOutput) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/
|
||||
"[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
|
||||
"{only_emit_if_all_present: true}",
|
||||
/*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
// No third input item.
|
||||
std::vector<int> inputs = {1, 2};
|
||||
AddInputItems(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, MixedVectorsAndItems) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/4,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<int> vector_0 = {1, 2};
|
||||
std::vector<int> vector_1 = {3, 4, 5};
|
||||
int item_0 = 6;
|
||||
int item_1 = 7;
|
||||
|
||||
AddInputVector(/*index*/ 0, vector_0, /*timestamp=*/1, &runner);
|
||||
AddInputVector(/*index*/ 1, vector_1, /*timestamp=*/1, &runner);
|
||||
AddInputItem(/*index*/ 2, item_0, /*timestamp=*/1, &runner);
|
||||
AddInputItem(/*index*/ 3, item_1, /*timestamp=*/1, &runner);
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3, 4, 5, 6, 7};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, MixedVectorsAndItemsAnother) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/4,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
int item_0 = 1;
|
||||
std::vector<int> vector_0 = {2, 3};
|
||||
std::vector<int> vector_1 = {4, 5, 6};
|
||||
int item_1 = 7;
|
||||
|
||||
AddInputItem(/*index*/ 0, item_0, /*timestamp=*/1, &runner);
|
||||
AddInputVector(/*index*/ 1, vector_0, /*timestamp=*/1, &runner);
|
||||
AddInputVector(/*index*/ 2, vector_1, /*timestamp=*/1, &runner);
|
||||
AddInputItem(/*index*/ 3, item_1, /*timestamp=*/1, &runner);
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3, 4, 5, 6, 7};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
void AddInputVectors(const std::vector<std::vector<float>>& inputs,
|
||||
int64 timestamp, CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
|
||||
@@ -17,12 +17,16 @@
|
||||
#include "mediapipe/calculators/core/constant_side_packet_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {} // namespace
|
||||
|
||||
// Generates an output side packet or multiple output side packets according to
|
||||
// the specified options.
|
||||
//
|
||||
@@ -71,6 +75,10 @@ class ConstantSidePacketCalculator : public CalculatorBase {
|
||||
packet.Set<bool>();
|
||||
} else if (packet_options.has_string_value()) {
|
||||
packet.Set<std::string>();
|
||||
} else if (packet_options.has_uint64_value()) {
|
||||
packet.Set<uint64>();
|
||||
} else if (packet_options.has_classification_list_value()) {
|
||||
packet.Set<ClassificationList>();
|
||||
} else {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"None of supported values were specified in options.");
|
||||
@@ -95,6 +103,11 @@ class ConstantSidePacketCalculator : public CalculatorBase {
|
||||
packet.Set(MakePacket<bool>(packet_options.bool_value()));
|
||||
} else if (packet_options.has_string_value()) {
|
||||
packet.Set(MakePacket<std::string>(packet_options.string_value()));
|
||||
} else if (packet_options.has_uint64_value()) {
|
||||
packet.Set(MakePacket<uint64>(packet_options.uint64_value()));
|
||||
} else if (packet_options.has_classification_list_value()) {
|
||||
packet.Set(MakePacket<ClassificationList>(
|
||||
packet_options.classification_list_value()));
|
||||
} else {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"None of supported values were specified in options.");
|
||||
|
||||
@@ -17,6 +17,9 @@ syntax = "proto2";
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
import "mediapipe/framework/formats/classification.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message ConstantSidePacketCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
@@ -29,6 +32,8 @@ message ConstantSidePacketCalculatorOptions {
|
||||
float float_value = 2;
|
||||
bool bool_value = 3;
|
||||
string string_value = 4;
|
||||
uint64 uint64_value = 5;
|
||||
ClassificationList classification_list_value = 6;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message DequantizeByteArrayCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional DequantizeByteArrayCalculatorOptions ext = 272316343;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
// Copyright 2019-2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -187,6 +187,14 @@ class GateCalculator : public CalculatorBase {
|
||||
last_gate_state_ = new_gate_state;
|
||||
|
||||
if (!allow) {
|
||||
// Close the output streams if the gate will be permanently closed.
|
||||
// Prevents buffering in calculators whose parents do no use SetOffset.
|
||||
for (int i = 0; i < num_data_streams_; ++i) {
|
||||
if (!cc->Outputs().Get("", i).IsClosed() &&
|
||||
use_side_packet_for_allow_disallow_) {
|
||||
cc->Outputs().Get("", i).Close();
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
// Copyright 2019-2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message GateCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional GateCalculatorOptions ext = 261754847;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
// Copyright 2019-2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/core/split_vector_calculator.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
@@ -233,5 +235,71 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
|
||||
kOutputName, output_fn);
|
||||
EXPECT_EQ(output, input_packets);
|
||||
}
|
||||
|
||||
constexpr char kDualInputGraphConfig[] = R"proto(
|
||||
input_stream: "input_0"
|
||||
input_stream: "input_1"
|
||||
input_stream: "input_select"
|
||||
output_stream: "test_output"
|
||||
node {
|
||||
calculator: "MuxCalculator"
|
||||
input_stream: "INPUT:0:input_0"
|
||||
input_stream: "INPUT:1:input_1"
|
||||
input_stream: "SELECT:input_select"
|
||||
output_stream: "OUTPUT:test_output"
|
||||
}
|
||||
)proto";
|
||||
|
||||
TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
|
||||
CalculatorGraphConfig config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
kDualInputGraphConfig);
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(config));
|
||||
|
||||
std::shared_ptr<int> output;
|
||||
MP_ASSERT_OK(
|
||||
graph.ObserveOutputStream("test_output", [&output](const Packet& p) {
|
||||
output = p.Get<std::shared_ptr<int>>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
auto one = std::make_shared<int>(1);
|
||||
auto two = std::make_shared<int>(2);
|
||||
auto three = std::make_shared<int>(3);
|
||||
std::weak_ptr<int> one_weak = one;
|
||||
std::weak_ptr<int> two_weak = two;
|
||||
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_0",
|
||||
MakePacket<std::shared_ptr<int>>(std::move(one)).At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_1",
|
||||
MakePacket<std::shared_ptr<int>>(std::move(two)).At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_1",
|
||||
MakePacket<std::shared_ptr<int>>(std::move(three)).At(Timestamp(1))));
|
||||
EXPECT_EQ(one, nullptr);
|
||||
EXPECT_EQ(two, nullptr);
|
||||
EXPECT_EQ(three, nullptr);
|
||||
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_select", MakePacket<int>(0).At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
EXPECT_EQ(*output, 1);
|
||||
EXPECT_NE(one_weak.lock(), nullptr);
|
||||
EXPECT_EQ(two_weak.lock(), nullptr);
|
||||
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_select", MakePacket<int>(1).At(Timestamp(1))));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
EXPECT_EQ(*output, 3);
|
||||
|
||||
MP_ASSERT_OK(graph.CloseAllInputStreams());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message PacketClonerCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional PacketClonerCalculatorOptions ext = 258872085;
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// For each non empty input packet, emits a single output packet containing a
|
||||
// boolean value "true", "false" in response to empty packets (a.k.a. timestamp
|
||||
// bound updates) This can be used to "flag" the presence of an arbitrary packet
|
||||
// type as input into a downstream calculator.
|
||||
//
|
||||
// Inputs:
|
||||
// PACKET - any type.
|
||||
//
|
||||
// Outputs:
|
||||
// PRESENCE - bool.
|
||||
// "true" if packet is not empty, "false" if there's timestamp bound update
|
||||
// instead.
|
||||
//
|
||||
// Examples:
|
||||
// node: {
|
||||
// calculator: "PacketPresenceCalculator"
|
||||
// input_stream: "PACKET:packet"
|
||||
// output_stream: "PRESENCE:presence"
|
||||
// }
|
||||
//
|
||||
// This calculator can be used in conjuction with GateCalculator in order to
|
||||
// allow/disallow processing. For instance:
|
||||
// node: {
|
||||
// calculator: "PacketPresenceCalculator"
|
||||
// input_stream: "PACKET:value"
|
||||
// output_stream: "PRESENCE:disallow_if_present"
|
||||
// }
|
||||
// node {
|
||||
// calculator: "GateCalculator"
|
||||
// input_stream: "image"
|
||||
// input_stream: "DISALLOW:disallow_if_present"
|
||||
// output_stream: "image_for_processing"
|
||||
// options: {
|
||||
// [mediapipe.GateCalculatorOptions.ext] {
|
||||
// empty_packets_as_allow: true
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class PacketPresenceCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag("PACKET").SetAny();
|
||||
cc->Outputs().Tag("PRESENCE").Set<bool>();
|
||||
// Process() function is invoked in response to input stream timestamp
|
||||
// bound updates.
|
||||
cc->SetProcessTimestampBounds(true);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) final {
|
||||
cc->Outputs()
|
||||
.Tag("PRESENCE")
|
||||
.AddPacket(MakePacket<bool>(!cc->Inputs().Tag("PACKET").IsEmpty())
|
||||
.At(cc->InputTimestamp()));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(PacketPresenceCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/framework/tool/sink.h"
|
||||
|
||||
namespace mediapipe {
|
||||
using ::testing::ElementsAre;
|
||||
using ::testing::Eq;
|
||||
using ::testing::Value;
|
||||
namespace {
|
||||
|
||||
MATCHER_P2(BoolPacket, value, timestamp, "") {
|
||||
return Value(arg.template Get<bool>(), Eq(value)) &&
|
||||
Value(arg.Timestamp(), Eq(timestamp));
|
||||
}
|
||||
|
||||
TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
std::vector<Packet> output_packets;
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: 'allow'
|
||||
input_stream: 'value'
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: 'value'
|
||||
input_stream: 'ALLOW:allow'
|
||||
output_stream: 'gated_value'
|
||||
}
|
||||
node {
|
||||
calculator: 'PacketPresenceCalculator'
|
||||
input_stream: 'PACKET:gated_value'
|
||||
output_stream: 'PRESENCE:presence'
|
||||
}
|
||||
)");
|
||||
tool::AddVectorSink("presence", &graph_config, &output_packets);
|
||||
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config, {}));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
auto send_packet = [&graph](int value, bool allow, Timestamp timestamp) {
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"value", MakePacket<int>(value).At(timestamp)));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"allow", MakePacket<bool>(allow).At(timestamp)));
|
||||
};
|
||||
|
||||
send_packet(10, false, Timestamp(10));
|
||||
MP_EXPECT_OK(graph.WaitUntilIdle());
|
||||
EXPECT_THAT(output_packets, ElementsAre(BoolPacket(false, Timestamp(10))));
|
||||
|
||||
output_packets.clear();
|
||||
send_packet(20, true, Timestamp(11));
|
||||
MP_EXPECT_OK(graph.WaitUntilIdle());
|
||||
EXPECT_THAT(output_packets, ElementsAre(BoolPacket(true, Timestamp(11))));
|
||||
|
||||
MP_EXPECT_OK(graph.CloseAllInputStreams());
|
||||
MP_EXPECT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message PacketResamplerCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional PacketResamplerCalculatorOptions ext = 95743844;
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message PacketThinnerCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional PacketThinnerCalculatorOptions ext = 288533508;
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message QuantizeFloatVectorCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional QuantizeFloatVectorCalculatorOptions ext = 259848061;
|
||||
|
||||
@@ -32,6 +32,9 @@ class SequenceShiftCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
if (cc->InputSidePackets().HasTag(kPacketOffsetTag)) {
|
||||
cc->InputSidePackets().Tag(kPacketOffsetTag).Set<int>();
|
||||
}
|
||||
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
@@ -41,6 +44,8 @@ class SequenceShiftCalculator : public CalculatorBase {
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
static constexpr const char* kPacketOffsetTag = "PACKET_OFFSET";
|
||||
|
||||
// A positive offset means we want a packet to be output with the timestamp of
|
||||
// a later packet. Stores packets waiting for their output timestamps and
|
||||
// outputs a single packet when the cache fills.
|
||||
@@ -70,6 +75,9 @@ REGISTER_CALCULATOR(SequenceShiftCalculator);
|
||||
::mediapipe::Status SequenceShiftCalculator::Open(CalculatorContext* cc) {
|
||||
packet_offset_ =
|
||||
cc->Options<mediapipe::SequenceShiftCalculatorOptions>().packet_offset();
|
||||
if (cc->InputSidePackets().HasTag(kPacketOffsetTag)) {
|
||||
packet_offset_ = cc->InputSidePackets().Tag(kPacketOffsetTag).Get<int>();
|
||||
}
|
||||
cache_size_ = abs(packet_offset_);
|
||||
// An offset of zero is a no-op, but someone might still request it.
|
||||
if (packet_offset_ == 0) {
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message SequenceShiftCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional SequenceShiftCalculatorOptions ext = 107633927;
|
||||
|
||||
@@ -99,6 +99,35 @@ TEST(SequenceShiftCalculatorTest, NegativeShift) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests using a side packet to specify the offset. Shifting by -2, i.e.,
|
||||
// output input[i] with timestamp[i - 2]. The first two packets should be
|
||||
// dropped.
|
||||
TEST(SequenceShiftCalculatorTest, SidePacketOffset) {
|
||||
CalculatorGraphConfig::Node node;
|
||||
node.set_calculator("SequenceShiftCalculator");
|
||||
node.add_input_stream("input");
|
||||
node.add_output_stream("output");
|
||||
node.add_input_side_packet("PACKET_OFFSET:packet_offset");
|
||||
|
||||
CalculatorRunner runner(node);
|
||||
AddPackets(&runner);
|
||||
runner.MutableSidePackets()->Tag("PACKET_OFFSET") = Adopt(new int(-2));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& input_packets =
|
||||
runner.MutableInputs()->Index(0).packets;
|
||||
const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
|
||||
ASSERT_EQ(10, input_packets.size());
|
||||
// Input packet[i] should be output with the timestamp of input packet[i - 2].
|
||||
// The first two packets are dropped. This means timestamps match between
|
||||
// input and output packets, but the data in the output packets come from
|
||||
// input_packets[i + 2].
|
||||
ASSERT_EQ(8, output_packets.size());
|
||||
for (int i = 0; i < output_packets.size(); ++i) {
|
||||
EXPECT_EQ(input_packets[i].Timestamp(), output_packets[i].Timestamp());
|
||||
EXPECT_EQ(input_packets[i + 2].Get<int>(), output_packets[i].Get<int>());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -16,10 +16,12 @@
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
@@ -46,15 +48,18 @@ typedef SplitVectorCalculator<TfLiteTensor, false>
|
||||
SplitTfLiteTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark, false>
|
||||
typedef SplitVectorCalculator<Tensor, true> SplitTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitTensorVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<mediapipe::NormalizedLandmark, false>
|
||||
SplitLandmarkVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::NormalizedLandmarkList, false>
|
||||
typedef SplitVectorCalculator<mediapipe::NormalizedLandmarkList, false>
|
||||
SplitNormalizedLandmarkListVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitNormalizedLandmarkListVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::NormalizedRect, false>
|
||||
typedef SplitVectorCalculator<mediapipe::NormalizedRect, false>
|
||||
SplitNormalizedRectVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
|
||||
|
||||
@@ -67,8 +72,12 @@ typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true>
|
||||
REGISTER_CALCULATOR(MovableSplitGlBufferVectorCalculator);
|
||||
#endif
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::Detection, false>
|
||||
typedef SplitVectorCalculator<mediapipe::Detection, false>
|
||||
SplitDetectionVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitDetectionVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<mediapipe::ClassificationList, false>
|
||||
SplitClassificationListVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitClassificationListVectorCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
// A Range {begin, end} specifies beginning ane ending indices to splice a
|
||||
// vector. A vector v is spliced to have elements v[begin:(end-1)], i.e., with
|
||||
// begin index inclusive and end index exclusive.
|
||||
|
||||
@@ -12,148 +12,78 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
exports_files(["LICENSE"])
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "opencv_image_encoder_calculator_proto",
|
||||
srcs = ["opencv_image_encoder_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "scale_image_calculator_proto",
|
||||
srcs = ["scale_image_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/framework/formats:image_format_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "set_alpha_calculator_proto",
|
||||
srcs = ["set_alpha_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "image_cropping_calculator_proto",
|
||||
srcs = ["image_cropping_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "bilateral_filter_calculator_proto",
|
||||
srcs = ["bilateral_filter_calculator.proto"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "recolor_calculator_proto",
|
||||
srcs = ["recolor_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/util:color_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "opencv_image_encoder_calculator_cc_proto",
|
||||
srcs = ["opencv_image_encoder_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [":opencv_image_encoder_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "opencv_encoded_image_to_image_frame_calculator_cc_proto",
|
||||
srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":opencv_encoded_image_to_image_frame_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "mask_overlay_calculator_cc_proto",
|
||||
srcs = ["mask_overlay_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":mask_overlay_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "scale_image_calculator_cc_proto",
|
||||
srcs = ["scale_image_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":scale_image_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "set_alpha_calculator_cc_proto",
|
||||
srcs = ["set_alpha_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":set_alpha_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "image_cropping_calculator_cc_proto",
|
||||
srcs = ["image_cropping_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":image_cropping_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "bilateral_filter_calculator_cc_proto",
|
||||
srcs = ["bilateral_filter_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [":bilateral_filter_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "recolor_calculator_cc_proto",
|
||||
srcs = ["recolor_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/util:color_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":recolor_calculator_proto"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "color_convert_calculator",
|
||||
srcs = ["color_convert_calculator.cc"],
|
||||
@@ -279,27 +209,17 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "image_transformation_calculator_proto",
|
||||
srcs = ["image_transformation_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/gpu:scale_mode_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "image_transformation_calculator_cc_proto",
|
||||
srcs = ["image_transformation_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/gpu:scale_mode_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":image_transformation_calculator_proto"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_transformation_calculator",
|
||||
srcs = ["image_transformation_calculator.cc"],
|
||||
@@ -560,32 +480,33 @@ cc_test(
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "mask_overlay_calculator_proto",
|
||||
srcs = ["mask_overlay_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "opencv_encoded_image_to_image_frame_calculator_proto",
|
||||
srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "feature_detector_calculator_proto",
|
||||
srcs = ["feature_detector_calculator.proto"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "feature_detector_calculator_cc_proto",
|
||||
srcs = ["feature_detector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":feature_detector_calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
@@ -630,3 +551,34 @@ cc_library(
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_file_properties_calculator",
|
||||
srcs = ["image_file_properties_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image_file_properties_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@easyexif",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "image_file_properties_calculator_test",
|
||||
srcs = ["image_file_properties_calculator_test.cc"],
|
||||
data = ["//mediapipe/calculators/image/testdata:test_images"],
|
||||
deps = [
|
||||
":image_file_properties_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats:image_file_properties_cc_proto",
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -107,7 +107,7 @@ class BilateralFilterCalculator : public CalculatorBase {
|
||||
GLuint program_ = 0;
|
||||
GLuint vao_;
|
||||
GLuint vbo_[2]; // vertex storage
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
};
|
||||
REGISTER_CALCULATOR(BilateralFilterCalculator);
|
||||
|
||||
|
||||
@@ -127,6 +127,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
|
||||
}
|
||||
|
||||
options_ = cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
|
||||
output_max_width_ =
|
||||
options_.has_output_max_width() ? options_.output_max_width() : FLT_MAX;
|
||||
output_max_height_ =
|
||||
options_.has_output_max_height() ? options_.output_max_height() : FLT_MAX;
|
||||
|
||||
if (use_gpu_) {
|
||||
#if !defined(MEDIAPIPE_DISABLE_GPU)
|
||||
@@ -234,20 +238,27 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
|
||||
cv::Mat src_points;
|
||||
cv::boxPoints(min_rect, src_points);
|
||||
|
||||
float output_width = min_rect.size.width;
|
||||
float output_height = min_rect.size.height;
|
||||
float scale = std::min({1.0f, output_max_width_ / output_width,
|
||||
output_max_height_ / output_height});
|
||||
output_width *= scale;
|
||||
output_height *= scale;
|
||||
|
||||
float dst_corners[8] = {0,
|
||||
min_rect.size.height - 1,
|
||||
output_height - 1,
|
||||
0,
|
||||
0,
|
||||
min_rect.size.width - 1,
|
||||
output_width - 1,
|
||||
0,
|
||||
min_rect.size.width - 1,
|
||||
min_rect.size.height - 1};
|
||||
output_width - 1,
|
||||
output_height - 1};
|
||||
cv::Mat dst_points = cv::Mat(4, 2, CV_32F, dst_corners);
|
||||
cv::Mat projection_matrix =
|
||||
cv::getPerspectiveTransform(src_points, dst_points);
|
||||
cv::Mat cropped_image;
|
||||
cv::warpPerspective(input_mat, cropped_image, projection_matrix,
|
||||
cv::Size(min_rect.size.width, min_rect.size.height),
|
||||
cv::Size(output_width, output_height),
|
||||
/* flags = */ 0,
|
||||
/* borderMode = */ border_mode);
|
||||
|
||||
@@ -439,6 +450,12 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
|
||||
|
||||
int width = static_cast<int>(std::round((col_max - col_min) * src_width));
|
||||
int height = static_cast<int>(std::round((row_max - row_min) * src_height));
|
||||
|
||||
float scale =
|
||||
std::min({1.0f, output_max_width_ / width, output_max_height_ / height});
|
||||
width *= scale;
|
||||
height *= scale;
|
||||
|
||||
// Minimum output dimension 1x1 prevents creation of textures with 0x0.
|
||||
*dst_width = std::max(1, width);
|
||||
*dst_height = std::max(1, height);
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#ifndef MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_
|
||||
|
||||
#include <float.h>
|
||||
|
||||
#include "mediapipe/calculators/image/image_cropping_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
|
||||
@@ -80,6 +82,8 @@ class ImageCroppingCalculator : public CalculatorBase {
|
||||
bool use_gpu_ = false;
|
||||
// Output texture corners (4) after transoformation in normalized coordinates.
|
||||
float transformed_points_[8];
|
||||
float output_max_width_ = FLT_MAX;
|
||||
float output_max_height_ = FLT_MAX;
|
||||
#if !defined(MEDIAPIPE_DISABLE_GPU)
|
||||
bool gpu_initialized_ = false;
|
||||
mediapipe::GlCalculatorHelper gpu_helper_;
|
||||
|
||||
@@ -51,4 +51,10 @@ message ImageCroppingCalculatorOptions {
|
||||
|
||||
// Specifies behaviour for crops that go beyond image borders.
|
||||
optional BorderMode border_mode = 8 [default = BORDER_ZERO];
|
||||
|
||||
// Specifies limits for the size of the output image. It will be scaled down,
|
||||
// preserving ratio, to fit within. These do not change which area of the
|
||||
// input is selected for cropping.
|
||||
optional int32 output_max_width = 9;
|
||||
optional int32 output_max_height = 10;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "exif.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image_file_properties.pb.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
// 35 MM sensor has dimensions 36 mm x 24 mm, so diagonal length is
|
||||
// sqrt(36^2 + 24^2).
|
||||
static const double SENSOR_DIAGONAL_35MM = std::sqrt(1872.0);
|
||||
|
||||
::mediapipe::StatusOr<double> ComputeFocalLengthInPixels(
|
||||
int image_width, int image_height, double focal_length_35mm,
|
||||
double focal_length_mm) {
|
||||
// TODO: Allow returning image file properties even when focal length
|
||||
// computation is not possible.
|
||||
if (image_width == 0 || image_height == 0) {
|
||||
return ::mediapipe::InternalError(
|
||||
"Image dimensions should be non-zero to compute focal length in "
|
||||
"pixels.");
|
||||
}
|
||||
if (focal_length_mm == 0) {
|
||||
return ::mediapipe::InternalError(
|
||||
"Focal length in mm should be non-zero to compute focal length in "
|
||||
"pixels.");
|
||||
}
|
||||
if (focal_length_35mm == 0) {
|
||||
return ::mediapipe::InternalError(
|
||||
"Focal length in 35 mm should be non-zero to compute focal length in "
|
||||
"pixels.");
|
||||
}
|
||||
// Derived from
|
||||
// https://en.wikipedia.org/wiki/35_mm_equivalent_focal_length#Calculation.
|
||||
/// Using focal_length_35mm = focal_length_mm * SENSOR_DIAGONAL_35MM /
|
||||
/// sensor_diagonal_mm, we can calculate the diagonal length of the sensor in
|
||||
/// millimeters i.e. sensor_diagonal_mm.
|
||||
double sensor_diagonal_mm =
|
||||
SENSOR_DIAGONAL_35MM / focal_length_35mm * focal_length_mm;
|
||||
// Note that for the following computations, the longer dimension is treated
|
||||
// as image width and the shorter dimension is treated as image height.
|
||||
int width = image_width;
|
||||
int height = image_height;
|
||||
if (image_height > image_width) {
|
||||
width = image_height;
|
||||
height = image_width;
|
||||
}
|
||||
double inv_aspect_ratio = (double)height / width;
|
||||
// Compute sensor width.
|
||||
/// Using Pythagoras theorem, sensor_width^2 + sensor_height^2 =
|
||||
/// sensor_diagonal_mm^2. We can substitute sensor_width / sensor_height with
|
||||
/// the aspect ratio calculated in pixels to compute the sensor width.
|
||||
double sensor_width = std::sqrt((sensor_diagonal_mm * sensor_diagonal_mm) /
|
||||
(1.0 + inv_aspect_ratio * inv_aspect_ratio));
|
||||
|
||||
// Compute focal length in pixels.
|
||||
double focal_length_pixels = width * focal_length_mm / sensor_width;
|
||||
return focal_length_pixels;
|
||||
}
|
||||
|
||||
::mediapipe::StatusOr<ImageFileProperties> GetImageFileProperites(
|
||||
const std::string& image_bytes) {
|
||||
easyexif::EXIFInfo result;
|
||||
int code = result.parseFrom(image_bytes);
|
||||
if (code) {
|
||||
return ::mediapipe::InternalError("Error parsing EXIF, code: " +
|
||||
std::to_string(code));
|
||||
}
|
||||
|
||||
ImageFileProperties properties;
|
||||
properties.set_image_width(result.ImageWidth);
|
||||
properties.set_image_height(result.ImageHeight);
|
||||
properties.set_focal_length_mm(result.FocalLength);
|
||||
properties.set_focal_length_35mm(result.FocalLengthIn35mm);
|
||||
|
||||
ASSIGN_OR_RETURN(auto focal_length_pixels,
|
||||
ComputeFocalLengthInPixels(properties.image_width(),
|
||||
properties.image_height(),
|
||||
properties.focal_length_35mm(),
|
||||
properties.focal_length_mm()));
|
||||
properties.set_focal_length_pixels(focal_length_pixels);
|
||||
|
||||
return properties;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// Calculator to extract EXIF information from an image file. The input is
|
||||
// a std::string containing raw byte data from a file, and the output is an
|
||||
// ImageFileProperties proto object with the relevant fields filled in.
|
||||
// The calculator accepts the input as a stream or a side packet, and can output
|
||||
// the result as a stream or a side packet. The calculator checks that if an
|
||||
// output stream is present, it outputs to that stream, and if not, it checks if
|
||||
// it can output to a side packet.
|
||||
//
|
||||
// Example config with input and output streams:
|
||||
// node {
|
||||
// calculator: "ImageFilePropertiesCalculator"
|
||||
// input_stream: "image_bytes"
|
||||
// output_stream: "image_properties"
|
||||
// }
|
||||
// Example config with input and output side packets:
|
||||
// node {
|
||||
// calculator: "ImageFilePropertiesCalculator"
|
||||
// input_side_packet: "image_bytes"
|
||||
// output_side_packet: "image_properties"
|
||||
// }
|
||||
class ImageFilePropertiesCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
if (cc->Inputs().NumEntries() != 0) {
|
||||
RET_CHECK(cc->Inputs().NumEntries() == 1);
|
||||
cc->Inputs().Index(0).Set<std::string>();
|
||||
} else {
|
||||
RET_CHECK(cc->InputSidePackets().NumEntries() == 1);
|
||||
cc->InputSidePackets().Index(0).Set<std::string>();
|
||||
}
|
||||
if (cc->Outputs().NumEntries() != 0) {
|
||||
RET_CHECK(cc->Outputs().NumEntries() == 1);
|
||||
cc->Outputs().Index(0).Set<::mediapipe::ImageFileProperties>();
|
||||
} else {
|
||||
RET_CHECK(cc->OutputSidePackets().NumEntries() == 1);
|
||||
cc->OutputSidePackets().Index(0).Set<::mediapipe::ImageFileProperties>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
if (cc->InputSidePackets().NumEntries() == 1) {
|
||||
const std::string& image_bytes =
|
||||
cc->InputSidePackets().Index(0).Get<std::string>();
|
||||
ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||
read_properties_ = true;
|
||||
}
|
||||
|
||||
if (read_properties_ && cc->OutputSidePackets().NumEntries() == 1) {
|
||||
cc->OutputSidePackets().Index(0).Set(
|
||||
MakePacket<ImageFileProperties>(properties_));
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (cc->Inputs().NumEntries() == 1) {
|
||||
if (cc->Inputs().Index(0).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
|
||||
ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||
read_properties_ = true;
|
||||
}
|
||||
if (read_properties_) {
|
||||
if (cc->Outputs().NumEntries() == 1) {
|
||||
cc->Outputs().Index(0).AddPacket(
|
||||
MakePacket<ImageFileProperties>(properties_)
|
||||
.At(cc->InputTimestamp()));
|
||||
} else {
|
||||
cc->OutputSidePackets().Index(0).Set(
|
||||
MakePacket<ImageFileProperties>(properties_)
|
||||
.At(::mediapipe::Timestamp::Unset()));
|
||||
}
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
ImageFileProperties properties_;
|
||||
bool read_properties_ = false;
|
||||
};
|
||||
REGISTER_CALCULATOR(ImageFilePropertiesCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,134 @@
|
||||
// Copyright 2018 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <math.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/formats/image_file_properties.pb.h"
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kImageFilePath[] =
|
||||
"/mediapipe/calculators/image/testdata/"
|
||||
"front_camera_pixel2.jpg";
|
||||
constexpr int kExpectedWidth = 2448;
|
||||
constexpr int kExpectedHeight = 3264;
|
||||
constexpr double kExpectedFocalLengthMm = 3.38;
|
||||
constexpr double kExpectedFocalLengthIn35Mm = 25;
|
||||
constexpr double kExpectedFocalLengthPixels = 2357.48;
|
||||
|
||||
double RoundToNDecimals(double value, int n) {
|
||||
return std::round(value * pow(10.0, n)) / pow(10.0, n);
|
||||
}
|
||||
|
||||
TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInStreams) {
|
||||
std::string image_filepath = file::JoinPath("./", kImageFilePath);
|
||||
std::string image_contents;
|
||||
MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "ImageFilePropertiesCalculator"
|
||||
input_stream: "image_bytes"
|
||||
output_stream: "properties"
|
||||
)");
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
MakePacket<std::string>(image_contents).At(Timestamp(0)));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const auto& outputs = runner.Outputs();
|
||||
ASSERT_EQ(1, outputs.NumEntries());
|
||||
const std::vector<Packet>& packets = outputs.Index(0).packets;
|
||||
ASSERT_EQ(1, packets.size());
|
||||
const auto& result = packets[0].Get<::mediapipe::ImageFileProperties>();
|
||||
EXPECT_EQ(kExpectedWidth, result.image_width());
|
||||
EXPECT_EQ(kExpectedHeight, result.image_height());
|
||||
EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm());
|
||||
EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm());
|
||||
EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels,
|
||||
RoundToNDecimals(result.focal_length_pixels(), /*n=*/2));
|
||||
}
|
||||
|
||||
TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInSidePackets) {
|
||||
std::string image_filepath = file::JoinPath("./", kImageFilePath);
|
||||
std::string image_contents;
|
||||
MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "ImageFilePropertiesCalculator"
|
||||
input_side_packet: "image_bytes"
|
||||
output_side_packet: "properties"
|
||||
)");
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Index(0) =
|
||||
MakePacket<std::string>(image_contents).At(Timestamp(0));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const auto& outputs = runner.OutputSidePackets();
|
||||
EXPECT_EQ(1, outputs.NumEntries());
|
||||
const auto& packet = outputs.Index(0);
|
||||
const auto& result = packet.Get<::mediapipe::ImageFileProperties>();
|
||||
EXPECT_EQ(kExpectedWidth, result.image_width());
|
||||
EXPECT_EQ(kExpectedHeight, result.image_height());
|
||||
EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm());
|
||||
EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm());
|
||||
EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels,
|
||||
RoundToNDecimals(result.focal_length_pixels(), /*n=*/2));
|
||||
}
|
||||
|
||||
TEST(ImageFilePropertiesCalculatorTest,
|
||||
ReadsFocalLengthFromJpegStreamToSidePacket) {
|
||||
std::string image_filepath = file::JoinPath("./", kImageFilePath);
|
||||
std::string image_contents;
|
||||
MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "ImageFilePropertiesCalculator"
|
||||
input_stream: "image_bytes"
|
||||
output_side_packet: "properties"
|
||||
)");
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
MakePacket<std::string>(image_contents).At(Timestamp(0)));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const auto& outputs = runner.OutputSidePackets();
|
||||
EXPECT_EQ(1, outputs.NumEntries());
|
||||
const auto& packet = outputs.Index(0);
|
||||
const auto& result = packet.Get<::mediapipe::ImageFileProperties>();
|
||||
EXPECT_EQ(kExpectedWidth, result.image_width());
|
||||
EXPECT_EQ(kExpectedHeight, result.image_height());
|
||||
EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm());
|
||||
EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm());
|
||||
EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels,
|
||||
RoundToNDecimals(result.focal_length_pixels(), /*n=*/2));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -519,7 +519,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
renderer = yuv_renderer_.get();
|
||||
src1 = gpu_helper_.CreateSourceTexture(input, 0);
|
||||
} else // NOLINT(readability/braces)
|
||||
#endif // iOS
|
||||
#endif // iOS
|
||||
{
|
||||
src1 = gpu_helper_.CreateSourceTexture(input);
|
||||
#if defined(TEXTURE_EXTERNAL_OES)
|
||||
@@ -531,7 +531,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
}
|
||||
renderer = ext_rgb_renderer_.get();
|
||||
} else // NOLINT(readability/braces)
|
||||
#endif // TEXTURE_EXTERNAL_OES
|
||||
#endif // TEXTURE_EXTERNAL_OES
|
||||
{
|
||||
if (!rgb_renderer_) {
|
||||
rgb_renderer_ = absl::make_unique<QuadRenderer>();
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
filegroup(
|
||||
name = "test_images",
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
load("@bazel_skylib//lib:selects.bzl", "selects")
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
selects.config_setting_group(
|
||||
name = "compute_shader_unavailable",
|
||||
match_any = [
|
||||
"//mediapipe/gpu:disable_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "inference_calculator_proto",
|
||||
srcs = ["inference_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "inference_calculator",
|
||||
srcs = ["inference_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
features = ["-layering_check"], # allow depending on inference_calculator_gpu_deps
|
||||
linkopts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-framework CoreVideo",
|
||||
"-framework MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":inference_calculator_cc_proto",
|
||||
"@com_google_absl//absl/memory",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/util:resource_util",
|
||||
"//mediapipe/util/tflite:config",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
|
||||
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
|
||||
"//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
] + select({
|
||||
":compute_shader_unavailable": [],
|
||||
"//conditions:default": [":inference_calculator_gpu_deps"],
|
||||
}) + select({
|
||||
"//conditions:default": [],
|
||||
"//mediapipe:android": [
|
||||
"//mediapipe/util/android/file/base",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/nnapi:nnapi_delegate",
|
||||
],
|
||||
}) + select({
|
||||
"//conditions:default": [
|
||||
"//mediapipe/util:cpu_util",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "inference_calculator_gpu_deps",
|
||||
deps = selects.with_or({
|
||||
"//mediapipe:ios": [
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
"//mediapipe/gpu:MPPMetalUtil",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/objc:mediapipe_framework_ios",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/metal:buffer_convert",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
|
||||
],
|
||||
"//mediapipe:macos": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/util/tflite:tflite_gpu_runner",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "tensor_converter_calculator_proto",
|
||||
srcs = ["tensor_converter_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensor_converter_calculator",
|
||||
srcs = ["tensor_converter_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
features = ["-layering_check"], # allow depending on tensor_converter_calculator_gpu_deps
|
||||
linkopts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-framework CoreVideo",
|
||||
"-framework MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":tensor_converter_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/util:resource_util",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensor_converter_calculator_gpu_deps",
|
||||
deps = select({
|
||||
"//mediapipe:android": [
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
"//mediapipe:ios": [
|
||||
"//mediapipe/gpu:MPPMetalUtil",
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
"//mediapipe/objc:mediapipe_framework_ios",
|
||||
],
|
||||
"//mediapipe:macos": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
"//mediapipe/gpu:shader_util",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensor_converter_calculator_test",
|
||||
srcs = ["tensor_converter_calculator_test.cc"],
|
||||
deps = [
|
||||
":tensor_converter_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "tensors_to_detections_calculator_proto",
|
||||
srcs = ["tensors_to_detections_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_detections_calculator",
|
||||
srcs = ["tensors_to_detections_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
features = ["-layering_check"], # allow depending on tensors_to_detections_calculator_gpu_deps
|
||||
linkopts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-framework CoreVideo",
|
||||
"-framework MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":tensors_to_detections_calculator_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
"@com_google_absl//absl/types:span",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats:location",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
] + select({
|
||||
":compute_shader_unavailable": [],
|
||||
"//conditions:default": [":tensors_to_detections_calculator_gpu_deps"],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_detections_calculator_gpu_deps",
|
||||
deps = select({
|
||||
"//mediapipe:ios": [
|
||||
"//mediapipe/gpu:MPPMetalUtil",
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
],
|
||||
"//mediapipe:macos": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "tensors_to_landmarks_calculator_proto",
|
||||
srcs = ["tensors_to_landmarks_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_landmarks_calculator",
|
||||
srcs = ["tensors_to_landmarks_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":tensors_to_landmarks_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_floats_calculator",
|
||||
srcs = ["tensors_to_floats_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_classification_calculator",
|
||||
srcs = ["tensors_to_classification_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":tensors_to_classification_calculator_cc_proto",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
"@com_google_absl//absl/types:span",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:location",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/util:resource_util",
|
||||
] + select({
|
||||
"//mediapipe:android": [
|
||||
"//mediapipe/util/android/file/base",
|
||||
],
|
||||
"//mediapipe:ios": [
|
||||
"//mediapipe/util/android/file/base",
|
||||
],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "tensors_to_classification_calculator_proto",
|
||||
srcs = ["tensors_to_classification_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensors_to_classification_calculator_test",
|
||||
srcs = ["tensors_to_classification_calculator_test.cc"],
|
||||
data = ["testdata/labelmap.txt"],
|
||||
deps = [
|
||||
":tensors_to_classification_calculator",
|
||||
":tensors_to_classification_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_googletest//:gtest_main",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_calculator",
|
||||
srcs = ["image_to_tensor_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
features = ["-layering_check"], # allow depending on image_to_tensor_calculator_gpu_deps
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":image_to_tensor_calculator_cc_proto",
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_converter_opencv",
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:port",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_calculator_gpu_deps",
|
||||
deps = select({
|
||||
"//mediapipe:android": [
|
||||
":image_to_tensor_converter_gl_buffer",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
"//mediapipe:apple": [
|
||||
":image_to_tensor_converter_metal",
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
"//conditions:default": [
|
||||
":image_to_tensor_converter_gl_buffer",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "image_to_tensor_calculator_proto",
|
||||
srcs = ["image_to_tensor_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "image_to_tensor_calculator_test",
|
||||
srcs = ["image_to_tensor_calculator_test.cc"],
|
||||
data = [
|
||||
"testdata/image_to_tensor/input.jpg",
|
||||
"testdata/image_to_tensor/large_sub_rect.png",
|
||||
"testdata/image_to_tensor/large_sub_rect_keep_aspect.png",
|
||||
"testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation.png",
|
||||
"testdata/image_to_tensor/medium_sub_rect_keep_aspect.png",
|
||||
"testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation.png",
|
||||
"testdata/image_to_tensor/medium_sub_rect_with_rotation.png",
|
||||
"testdata/image_to_tensor/noop_except_range.png",
|
||||
],
|
||||
deps = [
|
||||
":image_to_tensor_calculator",
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter",
|
||||
hdrs = ["image_to_tensor_converter.h"],
|
||||
deps = [
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter_opencv",
|
||||
srcs = ["image_to_tensor_converter_opencv.cc"],
|
||||
hdrs = ["image_to_tensor_converter_opencv.h"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
deps = [
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter_gl_buffer",
|
||||
srcs = ["image_to_tensor_converter_gl_buffer.cc"],
|
||||
hdrs = ["image_to_tensor_converter_gl_buffer.h"],
|
||||
deps = ["//mediapipe/framework:port"] + select({
|
||||
"//mediapipe:apple": [],
|
||||
"//conditions:default": [
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_utils",
|
||||
"@com_google_absl//absl/strings",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_buffer_format",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:types",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:command_queue",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_call",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_texture",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:request_gpu_info",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:variable",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl/converters:util",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter_gl_texture",
|
||||
srcs = ["image_to_tensor_converter_gl_texture.cc"],
|
||||
hdrs = ["image_to_tensor_converter_gl_texture.h"],
|
||||
deps = ["//mediapipe/framework:port"] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": [
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_utils",
|
||||
"@com_google_absl//absl/strings",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:shader_util",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter_metal",
|
||||
srcs = ["image_to_tensor_converter_metal.cc"],
|
||||
hdrs = ["image_to_tensor_converter_metal.h"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
linkopts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-framework CoreVideo",
|
||||
"-framework MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
deps = ["//mediapipe/framework:port"] + select({
|
||||
"//mediapipe:apple": [
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
"@com_google_absl//absl/strings",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_buffer_format",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:types",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_utils",
|
||||
srcs = ["image_to_tensor_utils.cc"],
|
||||
hdrs = ["image_to_tensor_utils.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"@com_google_absl//absl/types:optional",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "image_to_tensor_utils_test",
|
||||
srcs = ["image_to_tensor_utils_test.cc"],
|
||||
deps = [
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,275 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <array>
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_metal.h"
|
||||
#include "mediapipe/gpu/MPPMetalHelper.h"
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h"
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#else
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h"
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
namespace {
|
||||
constexpr char kInputCpu[] = "IMAGE";
|
||||
constexpr char kInputGpu[] = "IMAGE_GPU";
|
||||
constexpr char kOutputMatrix[] = "MATRIX";
|
||||
constexpr char kOutput[] = "TENSORS";
|
||||
constexpr char kInputNormRect[] = "NORM_RECT";
|
||||
constexpr char kOutputLetterboxPadding[] = "LETTERBOX_PADDING";
|
||||
} // namespace
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Converts image into Tensor, possibly with cropping, resizing and
|
||||
// normalization, according to specified inputs and options.
|
||||
//
|
||||
// Inputs:
|
||||
// IMAGE - ImageFrame [ImageFormat::SRGB/SRGBA]
|
||||
// Image to extract from.
|
||||
// IMAGE_GPU - GpuBuffer [GpuBufferFormat::kBGRA32]
|
||||
// Image to extract from.
|
||||
// (Either IMAGE or IMAGE_GPU has to be specified.)
|
||||
//
|
||||
// NORM_RECT - NormalizedRect @Optional
|
||||
// Describes region of image to extract.
|
||||
// @Optional: rect covering the whole image is used if not specified.
|
||||
//
|
||||
// Outputs:
|
||||
// TENSORS - std::vector<Tensor>
|
||||
// Vector containing a single Tensor populated with an extrated RGB image.
|
||||
// MATRIX - std::array<float, 16> @Optional
|
||||
// An std::array<float, 16> representing a 4x4 row-major-order matrix which
|
||||
// can be used to map a point on the output tensor to a point on the input
|
||||
// image.
|
||||
// LETTERBOX_PADDING - std::array<float, 4> @Optional
|
||||
// An std::array<float, 4> representing the letterbox padding from the 4
|
||||
// sides ([left, top, right, bottom]) of the output image, normalized to
|
||||
// [0.f, 1.f] by the output dimensions. The padding values are non-zero only
|
||||
// when the "keep_aspect_ratio" is true.
|
||||
//
|
||||
// For instance, when the input image is 10x10 (width x height) and the
|
||||
// output dimensions specified in the calculator option are 20x40 and
|
||||
// "keep_aspect_ratio" is true, the calculator scales the input image to
|
||||
// 20x20 and places it in the middle of the output image with an equal
|
||||
// padding of 10 pixels at the top and the bottom. The resulting array is
|
||||
// therefore [0.f, 0.25f, 0.f, 0.25f] (10/40 = 0.25f).
|
||||
//
|
||||
// Example:
|
||||
// node {
|
||||
// calculator: "ImageToTensorCalculator"
|
||||
// input_stream: "IMAGE:image" # or "IMAGE_GPU:image"
|
||||
// input_stream: "NORM_RECT:roi"
|
||||
// output_stream: "TENSORS:tensors"
|
||||
// output_stream: "MATRIX:matrix"
|
||||
// options {
|
||||
// [mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
// output_tensor_width: 256
|
||||
// output_tensor_height: 256
|
||||
// keep_aspect_ratio: false
|
||||
// output_tensor_float_range {
|
||||
// min: 0.0
|
||||
// max: 1.0
|
||||
// }
|
||||
// # gpu_origin: CONVENTIONAL # or TOP_LEFT
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class ImageToTensorCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
const auto& options =
|
||||
cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
|
||||
|
||||
RET_CHECK(options.has_output_tensor_float_range())
|
||||
<< "Output tensor range is required.";
|
||||
RET_CHECK_LT(options.output_tensor_float_range().min(),
|
||||
options.output_tensor_float_range().max())
|
||||
<< "Valid output tensor range is required.";
|
||||
RET_CHECK_GT(options.output_tensor_width(), 0)
|
||||
<< "Valid output tensor width is required.";
|
||||
RET_CHECK_GT(options.output_tensor_height(), 0)
|
||||
<< "Valid output tensor height is required.";
|
||||
|
||||
if (cc->Inputs().HasTag(kInputNormRect)) {
|
||||
cc->Inputs().Tag(kInputNormRect).Set<mediapipe::NormalizedRect>();
|
||||
}
|
||||
if (cc->Outputs().HasTag(kOutputLetterboxPadding)) {
|
||||
cc->Outputs().Tag(kOutputLetterboxPadding).Set<std::array<float, 4>>();
|
||||
}
|
||||
if (cc->Outputs().HasTag(kOutputMatrix)) {
|
||||
cc->Outputs().Tag(kOutputMatrix).Set<std::array<float, 16>>();
|
||||
}
|
||||
|
||||
const bool has_cpu_input = cc->Inputs().HasTag(kInputCpu);
|
||||
const bool has_gpu_input = cc->Inputs().HasTag(kInputGpu);
|
||||
RET_CHECK_EQ((has_cpu_input ? 1 : 0) + (has_gpu_input ? 1 : 0), 1)
|
||||
<< "Either CPU or GPU input is expected, not both.";
|
||||
|
||||
if (has_cpu_input) {
|
||||
cc->Inputs().Tag(kInputCpu).Set<mediapipe::ImageFrame>();
|
||||
} else if (has_gpu_input) {
|
||||
#if MEDIAPIPE_DISABLE_GPU
|
||||
return mediapipe::UnimplementedError("GPU processing is disabled");
|
||||
#else
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
|
||||
#else
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
cc->Inputs().Tag(kInputGpu).Set<mediapipe::GpuBuffer>();
|
||||
|
||||
#endif // MEDIAPIPE_DISABLE_GPU
|
||||
}
|
||||
cc->Outputs().Tag(kOutput).Set<std::vector<Tensor>>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) {
|
||||
// Makes sure outputs' next timestamp bound update is handled automatically
|
||||
// by the framework.
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
options_ = cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
|
||||
output_width_ = options_.output_tensor_width();
|
||||
output_height_ = options_.output_tensor_height();
|
||||
range_min_ = options_.output_tensor_float_range().min();
|
||||
range_max_ = options_.output_tensor_float_range().max();
|
||||
|
||||
if (cc->Inputs().HasTag(kInputCpu)) {
|
||||
ASSIGN_OR_RETURN(converter_, CreateOpenCvConverter(cc));
|
||||
} else {
|
||||
#if MEDIAPIPE_DISABLE_GPU
|
||||
return mediapipe::UnimplementedError("GPU processing is disabled");
|
||||
#else
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
ASSIGN_OR_RETURN(converter_, CreateMetalConverter(cc));
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
ASSIGN_OR_RETURN(converter_, CreateImageToGlBufferTensorConverter(
|
||||
cc, DoesInputStartAtBottom()));
|
||||
#else
|
||||
ASSIGN_OR_RETURN(converter_, CreateImageToGlTextureTensorConverter(
|
||||
cc, DoesInputStartAtBottom()));
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
|
||||
#endif // MEDIAPIPE_DISABLE_GPU
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) {
|
||||
const InputStreamShard& input = cc->Inputs().Tag(
|
||||
cc->Inputs().HasTag(kInputCpu) ? kInputCpu : kInputGpu);
|
||||
if (input.IsEmpty()) {
|
||||
// Timestamp bound update happens automatically. (See Open().)
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
absl::optional<mediapipe::NormalizedRect> norm_rect;
|
||||
if (cc->Inputs().HasTag(kInputNormRect)) {
|
||||
if (cc->Inputs().Tag(kInputNormRect).IsEmpty()) {
|
||||
// Timestamp bound update happens automatically. (See Open().)
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
norm_rect =
|
||||
cc->Inputs().Tag(kInputNormRect).Get<mediapipe::NormalizedRect>();
|
||||
if (norm_rect->width() == 0 && norm_rect->height() == 0) {
|
||||
// WORKAROUND: some existing graphs may use sentinel rects {width=0,
|
||||
// height=0, ...} quite often and calculator has to handle them
|
||||
// gracefully by updating timestamp bound instead of returning failure.
|
||||
// Timestamp bound update happens automatically. (See Open().)
|
||||
// NOTE: usage of sentinel rects should be avoided.
|
||||
DLOG(WARNING)
|
||||
<< "Updating timestamp bound in response to a sentinel rect";
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
}
|
||||
|
||||
const Packet& image_packet = input.Value();
|
||||
const Size& size = converter_->GetImageSize(image_packet);
|
||||
RotatedRect roi = GetRoi(size.width, size.height, norm_rect);
|
||||
ASSIGN_OR_RETURN(auto padding, PadRoi(options_.output_tensor_width(),
|
||||
options_.output_tensor_height(),
|
||||
options_.keep_aspect_ratio(), &roi));
|
||||
if (cc->Outputs().HasTag(kOutputLetterboxPadding)) {
|
||||
cc->Outputs()
|
||||
.Tag(kOutputLetterboxPadding)
|
||||
.AddPacket(MakePacket<std::array<float, 4>>(padding).At(
|
||||
cc->InputTimestamp()));
|
||||
}
|
||||
if (cc->Outputs().HasTag(kOutputMatrix)) {
|
||||
std::array<float, 16> matrix;
|
||||
GetRotatedSubRectToRectTransformMatrix(roi, size.width, size.height,
|
||||
/*flip_horizontaly=*/false,
|
||||
&matrix);
|
||||
cc->Outputs()
|
||||
.Tag(kOutputMatrix)
|
||||
.AddPacket(MakePacket<std::array<float, 16>>(std::move(matrix))
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
ASSIGN_OR_RETURN(
|
||||
Tensor tensor,
|
||||
converter_->Convert(image_packet, roi, {output_width_, output_height_},
|
||||
range_min_, range_max_));
|
||||
|
||||
std::vector<Tensor> result;
|
||||
result.push_back(std::move(tensor));
|
||||
cc->Outputs().Tag(kOutput).AddPacket(
|
||||
MakePacket<std::vector<Tensor>>(std::move(result))
|
||||
.At(cc->InputTimestamp()));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
bool DoesInputStartAtBottom() {
|
||||
return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
|
||||
}
|
||||
|
||||
std::unique_ptr<ImageToTensorConverter> converter_;
|
||||
mediapipe::ImageToTensorCalculatorOptions options_;
|
||||
int output_width_ = 0;
|
||||
int output_height_ = 0;
|
||||
float range_min_ = 0.0f;
|
||||
float range_max_ = 1.0f;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(ImageToTensorCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message GpuOrigin {
|
||||
enum Mode {
|
||||
DEFAULT = 0;
|
||||
|
||||
// OpenGL: bottom-left origin
|
||||
// Metal : top-left origin
|
||||
CONVENTIONAL = 1;
|
||||
|
||||
// OpenGL: top-left origin
|
||||
// Metal : top-left origin
|
||||
TOP_LEFT = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message ImageToTensorCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional ImageToTensorCalculatorOptions ext = 334361939;
|
||||
}
|
||||
|
||||
// Range of float values [min, max].
|
||||
// min, must be strictly less than max.
|
||||
message FloatRange {
|
||||
optional float min = 1;
|
||||
optional float max = 2;
|
||||
}
|
||||
|
||||
optional int32 output_tensor_width = 1;
|
||||
optional int32 output_tensor_height = 2;
|
||||
|
||||
// If true, image region will be extracted and copied into tensor keeping
|
||||
// region aspect ratio, which usually results in letterbox padding. Otherwise,
|
||||
// if false, image region is stretched to fill output tensor fully.
|
||||
optional bool keep_aspect_ratio = 3;
|
||||
|
||||
// Output tensor element range/type image pixels are converted to.
|
||||
oneof range {
|
||||
FloatRange output_tensor_float_range = 4;
|
||||
}
|
||||
|
||||
// For CONVENTIONAL mode for OpenGL, input image starts at bottom and needs
|
||||
// to be flipped vertically as tensors are expected to start at top.
|
||||
// (DEFAULT or unset interpreted as CONVENTIONAL.)
|
||||
optional GpuOrigin.Mode gpu_origin = 5;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user