Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 MediaPipe Teamandchuoling
|
f96eadd6df | ||
|
MediaPipe Teamandjqtang
|
a8d6ce95c4 | ||
|
MediaPipe Teamandchuoling
|
d53068fe2c | ||
|
MediaPipe Teamandchuoling
|
c828392681 | ||
|
MediaPipe Teamandchuoling
|
cccf6244d3 | ||
|
MediaPipe Teamandchuoling
|
8f69af91fe | ||
|
MediaPipe Teamandchuoling
|
a908d668c7 | ||
|
MediaPipe Teamandjqtang
|
1db91b550a | ||
|
MediaPipe Teamandchuoling
|
c0124fb83c | ||
|
MediaPipe Teamandchuoling
|
a7225b938a |
+1
-1
@@ -54,7 +54,7 @@ RUN pip3 install tf_slim
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# Install bazel
|
||||
ARG BAZEL_VERSION=2.0.0
|
||||
ARG BAZEL_VERSION=3.4.1
|
||||
RUN mkdir /bazel && \
|
||||
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
|
||||
azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||
|
||||
+7
-3
@@ -1,6 +1,10 @@
|
||||
global-exclude .git*
|
||||
global-exclude *_test.py
|
||||
|
||||
recursive-include mediapipe/models *.tflite *.txt
|
||||
recursive-include mediapipe/modules *.tflite *.txt
|
||||
recursive-include mediapipe/graphs *.binarypb
|
||||
include CONTRIBUTING.md
|
||||
include LICENSE
|
||||
include MANIFEST.in
|
||||
include README.md
|
||||
include requirements.txt
|
||||
|
||||
recursive-include mediapipe/modules *.tflite *.txt *.binarypb
|
||||
|
||||
@@ -8,46 +8,63 @@ nav_order: 1
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
## Cross-platform ML solutions made simple
|
||||
## Live ML anywhere
|
||||
|
||||
[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
|
||||
and developers to build world-class ML solutions and applications for mobile,
|
||||
desktop/cloud, web and IoT devices.
|
||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
||||
ML solutions for live and streaming media.
|
||||
|
||||
 | 
|
||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
||||
***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
 | 
|
||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
||||
|
||||
## ML solutions in MediaPipe
|
||||
|
||||
Face Detection | Face Mesh | Iris 🆕 | Hands | Pose 🆕
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :----:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose)
|
||||
Face Detection | Face Mesh | Iris | Hands | Pose | Hair Segmentation
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :---------------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/hair_segmentation)
|
||||
|
||||
Hair Segmentation | Object Detection | Box Tracking | Objectron | KNIFT
|
||||
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
||||
:----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
||||
|
||||
[]() | Android | iOS | Desktop | Python | Web | Coral
|
||||
:---------------------------------------------------------------------------- | :-----: | :-: | :-----: | :----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | | |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) 🆕 | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) 🆕 | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
[]() | Android | iOS | Desktop | Python | Web | Coral
|
||||
:---------------------------------------------------------------------------------------- | :-----: | :-: | :-----: | :----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
|
||||
## MediaPipe in Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described in:
|
||||
|
||||
* [MediaPipe Face Mesh](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/face_mesh_py_colab)
|
||||
* [MediaPipe Hands](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/hands_py_colab)
|
||||
* [MediaPipe Pose](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/pose_py_colab)
|
||||
|
||||
## MediaPipe on the Web
|
||||
|
||||
@@ -88,7 +105,11 @@ run code search using
|
||||
|
||||
## Publications
|
||||
|
||||
* [BlazePose - On-device Real-time Body Pose Tracking](https://mediapipe.page.link/blazepose-blog)
|
||||
* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
|
||||
in Google Developers Blog
|
||||
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
|
||||
in Google Developers Blog
|
||||
* [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
in Google AI Blog
|
||||
* [MediaPipe Iris: Real-time Eye Tracking and Depth Estimation](https://ai.googleblog.com/2020/08/mediapipe-iris-real-time-iris-tracking.html)
|
||||
in Google AI Blog
|
||||
|
||||
@@ -10,7 +10,7 @@ http_archive(
|
||||
sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
|
||||
)
|
||||
load("@bazel_skylib//lib:versions.bzl", "versions")
|
||||
versions.check(minimum_bazel_version = "2.0.0")
|
||||
versions.check(minimum_bazel_version = "3.4.0")
|
||||
|
||||
|
||||
# ABSL cpp library lts_2020_02_25
|
||||
@@ -36,6 +36,19 @@ http_archive(
|
||||
urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "rules_foreign_cc",
|
||||
strip_prefix = "rules_foreign_cc-master",
|
||||
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/master.zip",
|
||||
)
|
||||
|
||||
load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
|
||||
|
||||
rules_foreign_cc_dependencies()
|
||||
|
||||
# This is used to select all contents of the archives for CMake-based packages to give CMake access to them.
|
||||
all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])"""
|
||||
|
||||
# GoogleTest/GoogleMock framework. Used by most unit-tests.
|
||||
# Last updated 2020-06-30.
|
||||
http_archive(
|
||||
@@ -68,14 +81,23 @@ http_archive(
|
||||
url = "https://github.com/gflags/gflags/archive/v2.2.2.zip",
|
||||
)
|
||||
|
||||
# glog v0.3.5
|
||||
# TODO: Migrate MediaPipe to use com_github_glog_glog on all platforms.
|
||||
# 2020-08-21
|
||||
http_archive(
|
||||
name = "com_github_glog_glog_v_0_3_5",
|
||||
url = "https://github.com/google/glog/archive/v0.3.5.zip",
|
||||
sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
|
||||
strip_prefix = "glog-0.3.5",
|
||||
build_file = "@//third_party:glog.BUILD",
|
||||
name = "com_github_glog_glog",
|
||||
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
|
||||
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
|
||||
urls = [
|
||||
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
|
||||
],
|
||||
)
|
||||
http_archive(
|
||||
name = "com_github_glog_glog_no_gflags",
|
||||
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
|
||||
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
|
||||
build_file = "@//third_party:glog_no_gflags.BUILD",
|
||||
urls = [
|
||||
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
|
||||
],
|
||||
patches = [
|
||||
"@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
|
||||
],
|
||||
@@ -84,16 +106,6 @@ http_archive(
|
||||
],
|
||||
)
|
||||
|
||||
# 2020-02-16
|
||||
http_archive(
|
||||
name = "com_github_glog_glog",
|
||||
strip_prefix = "glog-3ba8976592274bc1f907c402ce22558011d6fc5e",
|
||||
sha256 = "feca3c7e29a693cab7887409756d89d342d4a992d54d7c5599bebeae8f7b50be",
|
||||
urls = [
|
||||
"https://github.com/google/glog/archive/3ba8976592274bc1f907c402ce22558011d6fc5e.zip",
|
||||
],
|
||||
)
|
||||
|
||||
# easyexif
|
||||
http_archive(
|
||||
name = "easyexif",
|
||||
@@ -169,6 +181,13 @@ http_archive(
|
||||
sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "opencv",
|
||||
build_file_content = all_content,
|
||||
strip_prefix = "opencv-3.4.10",
|
||||
urls = ["https://github.com/opencv/opencv/archive/3.4.10.tar.gz"],
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
@@ -184,13 +203,13 @@ new_local_repository(
|
||||
new_local_repository(
|
||||
name = "macos_opencv",
|
||||
build_file = "@//third_party:opencv_macos.BUILD",
|
||||
path = "/usr",
|
||||
path = "/usr/local/opt/opencv@3",
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "macos_ffmpeg",
|
||||
build_file = "@//third_party:ffmpeg_macos.BUILD",
|
||||
path = "/usr",
|
||||
path = "/usr/local/opt/ffmpeg",
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
@@ -301,24 +320,27 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
|
||||
maven_install(
|
||||
name = "maven",
|
||||
artifacts = [
|
||||
"junit:junit:4.12",
|
||||
"androidx.test.espresso:espresso-core:3.1.1",
|
||||
"org.hamcrest:hamcrest-library:1.3",
|
||||
"androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
"androidx.lifecycle:lifecycle-common:2.2.0",
|
||||
"androidx.annotation:annotation:aar:1.1.0",
|
||||
"androidx.appcompat:appcompat:aar:1.1.0-rc01",
|
||||
"androidx.camera:camera-core:aar:1.0.0-alpha06",
|
||||
"androidx.camera:camera-camera2:aar:1.0.0-alpha06",
|
||||
"androidx.camera:camera-core:1.0.0-beta10",
|
||||
"androidx.camera:camera-camera2:1.0.0-beta10",
|
||||
"androidx.camera:camera-lifecycle:1.0.0-beta10",
|
||||
"androidx.constraintlayout:constraintlayout:aar:1.1.3",
|
||||
"androidx.core:core:aar:1.1.0-rc03",
|
||||
"androidx.legacy:legacy-support-v4:aar:1.0.0",
|
||||
"androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
|
||||
"androidx.test.espresso:espresso-core:3.1.1",
|
||||
"com.github.bumptech.glide:glide:4.11.0",
|
||||
"com.google.android.material:material:aar:1.0.0-rc01",
|
||||
"com.google.code.findbugs:jsr305:3.0.2",
|
||||
"com.google.flogger:flogger-system-backend:0.3.1",
|
||||
"com.google.flogger:flogger:0.3.1",
|
||||
"com.google.guava:guava:27.0.1-android",
|
||||
"com.google.guava:listenablefuture:1.0",
|
||||
"junit:junit:4.12",
|
||||
"org.hamcrest:hamcrest-library:1.3",
|
||||
],
|
||||
repositories = [
|
||||
"https://jcenter.bintray.com",
|
||||
@@ -342,13 +364,12 @@ http_archive(
|
||||
)
|
||||
|
||||
#Tensorflow repo should always go after the other external dependencies.
|
||||
# 2020-05-11
|
||||
_TENSORFLOW_GIT_COMMIT = "7c09d15f9fcc14343343c247ebf5b8e0afe3e4aa"
|
||||
_TENSORFLOW_SHA256= "673d00cbd2676ae43df1993e0d28c10b5ffbe96d9e2ab29f88a77b43c0211299"
|
||||
# 2020-10-30
|
||||
_TENSORFLOW_GIT_COMMIT = "84384703c0d8b502e33ff6fd7eefd219dca5ff8e"
|
||||
_TENSORFLOW_SHA256= "23fb322fc15a20f7a7838d9a31f8b16f60700a494ea654311a0aa8621769df98"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
],
|
||||
patches = [
|
||||
|
||||
+28
-26
@@ -93,38 +93,40 @@ for app in ${apps}; do
|
||||
|
||||
echo "=== Target: ${target}"
|
||||
|
||||
if [[ $install_only == false ]]; then
|
||||
bazel_flags=("${default_bazel_flags[@]}")
|
||||
bazel_flags+=(${target})
|
||||
if [[ $strip == true ]]; then
|
||||
bazel_flags+=(--linkopt=-s)
|
||||
fi
|
||||
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_4
|
||||
fi
|
||||
bazel "${bazel_flags[@]}"
|
||||
cp -f "${bin}" "${apk}"
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_3
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${app_name} == "objectdetection3d" ]]; then
|
||||
orig_apk=${apk}
|
||||
apk="${out_dir}/${target_name}_shoes.apk"
|
||||
cp -f "${orig_apk}" "${apk}"
|
||||
apks+=(${apk})
|
||||
|
||||
apk="${out_dir}/${target_name}_chairs.apk"
|
||||
categories=("shoe" "chair" "cup" "camera" "shoe_1stage" "chair_1stage")
|
||||
for category in ${categories[@]}; do
|
||||
apk="${out_dir}/${target_name}_${category}.apk"
|
||||
if [[ $install_only == false ]]; then
|
||||
bazel_flags_extended=("${bazel_flags[@]}")
|
||||
if [[ ${category} != "shoe" ]]; then
|
||||
bazel_flags_extended+=(--define ${category}=true)
|
||||
fi
|
||||
echo "bazel ${bazel_flags_extended[@]}"
|
||||
bazel "${bazel_flags_extended[@]}"
|
||||
cp -f "${bin}" "${apk}"
|
||||
fi
|
||||
apks+=(${apk})
|
||||
done
|
||||
else
|
||||
if [[ $install_only == false ]]; then
|
||||
bazel_flags+=(--define chair=true)
|
||||
bazel_flags=("${default_bazel_flags[@]}")
|
||||
bazel_flags+=(${target})
|
||||
if [[ $strip == true ]]; then
|
||||
bazel_flags+=(--linkopt=-s)
|
||||
fi
|
||||
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_4
|
||||
fi
|
||||
bazel "${bazel_flags[@]}"
|
||||
cp -f "${bin}" "${apk}"
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_3
|
||||
fi
|
||||
fi
|
||||
apks+=(${apk})
|
||||
fi
|
||||
|
||||
apks+=(${apk})
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# =========================================================================
|
||||
#
|
||||
# Script to build/run all MediaPipe desktop example apps (with webcam input).
|
||||
#
|
||||
# To build and run all apps and store them in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir
|
||||
# Omitting -d and the associated directory saves all generated apps in the
|
||||
# current directory.
|
||||
# To build all apps and store them in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir -b
|
||||
# Omitting -d and the associated directory saves all generated apps in the
|
||||
# current directory.
|
||||
# To run all apps already stored in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir -r
|
||||
# Omitting -d and the associated directory assumes all apps are in the current
|
||||
# directory.
|
||||
|
||||
set -e
|
||||
|
||||
out_dir="."
|
||||
build_only=false
|
||||
run_only=false
|
||||
app_dir="mediapipe/examples/desktop"
|
||||
bin_dir="bazel-bin"
|
||||
declare -a default_bazel_flags=(build -c opt --define MEDIAPIPE_DISABLE_GPU=1)
|
||||
|
||||
while [[ -n $1 ]]; do
|
||||
case $1 in
|
||||
-d)
|
||||
shift
|
||||
out_dir=$1
|
||||
;;
|
||||
-b)
|
||||
build_only=true
|
||||
;;
|
||||
-r)
|
||||
run_only=true
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported input argument $1."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
echo "app_dir: $app_dir"
|
||||
echo "out_dir: $out_dir"
|
||||
|
||||
declare -a bazel_flags
|
||||
|
||||
apps="${app_dir}/*"
|
||||
for app in ${apps}; do
|
||||
if [[ -d "${app}" ]]; then
|
||||
target_name=${app##*/}
|
||||
if [[ "${target_name}" == "autoflip" ||
|
||||
"${target_name}" == "hello_world" ||
|
||||
"${target_name}" == "media_sequence" ||
|
||||
"${target_name}" == "template_matching" ||
|
||||
"${target_name}" == "youtube8m" ]]; then
|
||||
continue
|
||||
fi
|
||||
target="${app}:${target_name}_cpu"
|
||||
|
||||
echo "=== Target: ${target}"
|
||||
|
||||
if [[ $run_only == false ]]; then
|
||||
bazel_flags=("${default_bazel_flags[@]}")
|
||||
bazel_flags+=(${target})
|
||||
|
||||
bazel "${bazel_flags[@]}"
|
||||
cp -f "${bin_dir}/${app}/"*"_cpu" "${out_dir}"
|
||||
fi
|
||||
if [[ $build_only == false ]]; then
|
||||
if [[ ${target_name} == "object_tracking" ]]; then
|
||||
graph_name="tracking/object_detection_tracking"
|
||||
elif [[ ${target_name} == "upper_body_pose_tracking" ]]; then
|
||||
graph_name="pose_tracking/upper_body_pose_tracking"
|
||||
else
|
||||
graph_name="${target_name}/${target_name}"
|
||||
fi
|
||||
if [[ ${target_name} == "iris_tracking" ||
|
||||
${target_name} == "upper_body_pose_tracking" ]]; then
|
||||
graph_suffix="cpu"
|
||||
else
|
||||
graph_suffix="desktop_live"
|
||||
fi
|
||||
GLOG_logtostderr=1 "${out_dir}/${target_name}_cpu" \
|
||||
--calculator_graph_config_file=mediapipe/graphs/"${graph_name}_${graph_suffix}.pbtxt"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
+2
-2
@@ -22,8 +22,8 @@ aux_links:
|
||||
# Footer content appears at the bottom of every page's main content
|
||||
footer_content: "© 2020 GOOGLE LLC | <a href=\"https://policies.google.com/privacy\">PRIVACY POLICY</a> | <a href=\"https://policies.google.com/terms\">TERMS OF SERVICE</a>"
|
||||
|
||||
# Color scheme currently only supports "dark" or nil (default)
|
||||
color_scheme: nil
|
||||
# Color scheme currently only supports "dark", "light"/nil (default), or a custom scheme that you define
|
||||
color_scheme: mediapipe
|
||||
|
||||
# Google Analytics Tracking (optional)
|
||||
ga_tracking: UA-140696581-2
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
$link-color: #0097A7;
|
||||
@@ -132,9 +132,10 @@ each project.
|
||||
implementation 'com.google.guava:guava:27.0.1-android'
|
||||
implementation 'com.google.protobuf:protobuf-java:3.11.4'
|
||||
// CameraX core library
|
||||
def camerax_version = "1.0.0-alpha06"
|
||||
def camerax_version = "1.0.0-beta10"
|
||||
implementation "androidx.camera:camera-core:$camerax_version"
|
||||
implementation "androidx.camera:camera-camera2:$camerax_version"
|
||||
implementation "androidx.camera:camera-lifecycle:$camerax_version"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -425,7 +425,15 @@ Note: This currently works only on Linux, and please first follow
|
||||
|
||||
## Python
|
||||
|
||||
### Prerequisite
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described in, for instance,
|
||||
[Python section in MediaPipe Pose](../solutions/pose.md#python) and in this
|
||||
[colab](https://mediapipe.page.link/pose_py_colab).
|
||||
|
||||
Follow the steps below only if you have local changes and need to build the
|
||||
Python package from source. Otherwise, we strongly encourage our users to simply
|
||||
run `pip install mediapipe`, more convenient and much faster.
|
||||
|
||||
1. Make sure that Bazel and OpenCV are correctly installed and configured for
|
||||
MediaPipe. Please see [Installation](./install.md) for how to setup Bazel
|
||||
@@ -445,50 +453,23 @@ Note: This currently works only on Linux, and please first follow
|
||||
$ brew install protobuf
|
||||
```
|
||||
|
||||
### Set up Python virtual environment.
|
||||
|
||||
1. Activate a Python virtual environment.
|
||||
3. Activate a Python virtual environment.
|
||||
|
||||
```bash
|
||||
$ python3 -m venv mp_env && source mp_env/bin/activate
|
||||
```
|
||||
|
||||
2. In the virtual environment, go to the MediaPipe repo directory.
|
||||
4. In the virtual environment, go to the MediaPipe repo directory.
|
||||
|
||||
3. Install the required Python packages.
|
||||
5. Install the required Python packages.
|
||||
|
||||
```bash
|
||||
(mp_env)mediapipe$ pip3 install -r requirements.txt
|
||||
```
|
||||
|
||||
4. Generate and install MediaPipe package.
|
||||
6. Generate and install MediaPipe package.
|
||||
|
||||
```bash
|
||||
(mp_env)mediapipe$ python3 setup.py gen_protos
|
||||
(mp_env)mediapipe$ python3 setup.py install
|
||||
(mp_env)mediapipe$ python3 setup.py install --link-opencv
|
||||
```
|
||||
|
||||
### Run in Python interpreter
|
||||
|
||||
Make sure you are not in the MediaPipe repo directory.
|
||||
|
||||
Using [MediaPipe Pose](../solutions/pose.md) as an example:
|
||||
|
||||
```bash
|
||||
(mp_env)$ python3
|
||||
>>> import mediapipe as mp
|
||||
>>> pose_tracker = mp.examples.UpperBodyPoseTracker()
|
||||
|
||||
# For image input
|
||||
>>> pose_landmarks, _ = pose_tracker.run(input_file='/path/to/input/file', output_file='/path/to/output/file')
|
||||
>>> pose_landmarks, annotated_image = pose_tracker.run(input_file='/path/to/file')
|
||||
|
||||
# For live camera input
|
||||
# (Press Esc within the output image window to stop the run or let it self terminate after 30 seconds.)
|
||||
>>> pose_tracker.run_live()
|
||||
|
||||
# Close the tracker.
|
||||
>>> pose_tracker.close()
|
||||
```
|
||||
|
||||
Tip: Use command `deactivate` to exit the Python virtual environment.
|
||||
|
||||
@@ -446,8 +446,8 @@ visible so that we can start seeing frames from the `previewFrameTexture`.
|
||||
However, before starting the camera, we need to decide which camera we want to
|
||||
use. [`CameraXPreviewHelper`] inherits from [`CameraHelper`] which provides two
|
||||
options, `FRONT` and `BACK`. We can pass in the decision from the `BUILD` file
|
||||
as metadata such that no code change is required to build a another version of
|
||||
the app using a different camera.
|
||||
as metadata such that no code change is required to build another version of the
|
||||
app using a different camera.
|
||||
|
||||
Assuming we want to use `BACK` camera to perform edge detection on a live scene
|
||||
that we view from the camera, add the metadata into `AndroidManifest.xml`:
|
||||
|
||||
@@ -12,7 +12,7 @@ nav_order: 1
|
||||
{:toc}
|
||||
---
|
||||
|
||||
Note: To interoperate with OpenCV, OpenCV 3.x and above are preferred. OpenCV
|
||||
Note: To interoperate with OpenCV, OpenCV 3.x to 4.1 are preferred. OpenCV
|
||||
2.x currently works but interoperability support may be deprecated in the
|
||||
future.
|
||||
|
||||
@@ -42,16 +42,16 @@ apps, see these [instructions](./building_examples.md#ios).
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
|
||||
to install Bazel 2.0 or higher.
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
For Nvidia Jetson and Raspberry Pi devices with ARM Ubuntu, Bazel needs to
|
||||
be built from source.
|
||||
|
||||
```bash
|
||||
# For Bazel 3.0.0
|
||||
wget https://github.com/bazelbuild/bazel/releases/download/3.0.0/bazel-3.0.0-dist.zip
|
||||
# For Bazel 3.4.0
|
||||
wget https://github.com/bazelbuild/bazel/releases/download/3.4.0/bazel-3.4.0-dist.zip
|
||||
sudo apt-get install build-essential openjdk-8-jdk python zip unzip
|
||||
unzip bazel-3.0.0-dist.zip
|
||||
unzip bazel-3.4.0-dist.zip
|
||||
env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
|
||||
sudo cp output/bazel /usr/local/bin/
|
||||
```
|
||||
@@ -221,7 +221,7 @@ build issues.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
|
||||
to install Bazel 2.0 or higher.
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
3. Install OpenCV.
|
||||
|
||||
@@ -356,7 +356,7 @@ build issues.
|
||||
|
||||
Option 2. Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
|
||||
to install Bazel 2.0 or higher.
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
4. Install OpenCV and FFmpeg.
|
||||
|
||||
@@ -427,7 +427,6 @@ build issues.
|
||||
linkstatic = 1,
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
```
|
||||
|
||||
5. Make sure that Python 3 and the Python "six" library are installed.
|
||||
@@ -506,7 +505,7 @@ next section.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
|
||||
to install Bazel 2.0 or higher.
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
6. Set Bazel variables.
|
||||
|
||||
@@ -567,7 +566,6 @@ next section.
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.280613 1200 hello_world.cc:56] Hello World!
|
||||
|
||||
```
|
||||
|
||||
If you run into a build error, please read
|
||||
@@ -607,14 +605,14 @@ cameras. Alternatively, you use a video file as input.
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
|
||||
https://storage.googleapis.com/bazel/3.0.0/release/bazel-3.0.0-installer-linux-x86_64.sh && \
|
||||
sudo mkdir -p /usr/local/bazel/3.0.0 && \
|
||||
chmod 755 bazel-3.0.0-installer-linux-x86_64.sh && \
|
||||
sudo ./bazel-3.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.0.0 && \
|
||||
source /usr/local/bazel/3.0.0/lib/bazel/bin/bazel-complete.bash
|
||||
https://storage.googleapis.com/bazel/3.4.0/release/bazel-3.4.0-installer-linux-x86_64.sh && \
|
||||
sudo mkdir -p /usr/local/bazel/3.4.0 && \
|
||||
chmod 755 bazel-3.4.0-installer-linux-x86_64.sh && \
|
||||
sudo ./bazel-3.4.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.4.0 && \
|
||||
source /usr/local/bazel/3.4.0/lib/bazel/bin/bazel-complete.bash
|
||||
|
||||
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.0.0/lib/bazel/bin/bazel version && \
|
||||
alias bazel='/usr/local/bazel/3.0.0/lib/bazel/bin/bazel'
|
||||
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.4.0/lib/bazel/bin/bazel version && \
|
||||
alias bazel='/usr/local/bazel/3.4.0/lib/bazel/bin/bazel'
|
||||
```
|
||||
|
||||
6. Checkout MediaPipe repository.
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 524 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 808 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 7.1 MiB After Width: | Height: | Size: 6.7 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 925 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 889 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.5 MiB After Width: | Height: | Size: 923 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 670 KiB After Width: | Height: | Size: 744 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 625 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.8 MiB After Width: | Height: | Size: 968 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
+48
-27
@@ -8,46 +8,63 @@ nav_order: 1
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
## Cross-platform ML solutions made simple
|
||||
## Live ML anywhere
|
||||
|
||||
[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
|
||||
and developers to build world-class ML solutions and applications for mobile,
|
||||
desktop/cloud, web and IoT devices.
|
||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
||||
ML solutions for live and streaming media.
|
||||
|
||||
 | 
|
||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
||||
***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
 | 
|
||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
||||
|
||||
## ML solutions in MediaPipe
|
||||
|
||||
Face Detection | Face Mesh | Iris 🆕 | Hands | Pose 🆕
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :----:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose)
|
||||
Face Detection | Face Mesh | Iris | Hands | Pose | Hair Segmentation
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :---------------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/hair_segmentation)
|
||||
|
||||
Hair Segmentation | Object Detection | Box Tracking | Objectron | KNIFT
|
||||
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
||||
:----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
||||
|
||||
[]() | Android | iOS | Desktop | Python | Web | Coral
|
||||
:---------------------------------------------------------------------------- | :-----: | :-: | :-----: | :----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | | |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) 🆕 | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) 🆕 | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
[]() | Android | iOS | Desktop | Python | Web | Coral
|
||||
:---------------------------------------------------------------------------------------- | :-----: | :-: | :-----: | :----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
|
||||
## MediaPipe in Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described in:
|
||||
|
||||
* [MediaPipe Face Mesh](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/face_mesh_py_colab)
|
||||
* [MediaPipe Hands](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/hands_py_colab)
|
||||
* [MediaPipe Pose](../solutions/pose.md#python) and
|
||||
[colab](https://mediapipe.page.link/pose_py_colab)
|
||||
|
||||
## MediaPipe on the Web
|
||||
|
||||
@@ -88,7 +105,11 @@ run code search using
|
||||
|
||||
## Publications
|
||||
|
||||
* [BlazePose - On-device Real-time Body Pose Tracking](https://mediapipe.page.link/blazepose-blog)
|
||||
* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
|
||||
in Google Developers Blog
|
||||
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
|
||||
in Google Developers Blog
|
||||
* [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
in Google AI Blog
|
||||
* [MediaPipe Iris: Real-time Eye Tracking and Depth Estimation](https://ai.googleblog.com/2020/08/mediapipe-iris-real-time-iris-tracking.html)
|
||||
in Google AI Blog
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: AutoFlip (Saliency-aware Video Cropping)
|
||||
parent: Solutions
|
||||
nav_order: 11
|
||||
nav_order: 12
|
||||
---
|
||||
|
||||
# AutoFlip: Saliency-aware Video Cropping
|
||||
|
||||
@@ -102,9 +102,4 @@ to cross-compile and run MediaPipe examples on the
|
||||
[BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs](https://arxiv.org/abs/1907.05047)
|
||||
([presentation](https://docs.google.com/presentation/d/1YCtASfnYyZtH-41QvnW5iZxELFnf0MF-pPWSLGj8yjQ/present?slide=id.g5bc8aeffdd_1_0))
|
||||
([poster](https://drive.google.com/file/d/1u6aB6wxDY7X2TmeUUKgFydulNtXkb3pu/view))
|
||||
* For front-facing/selfie camera:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
|
||||
[TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
|
||||
* For back-facing camera:
|
||||
[TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
|
||||
* [Model card](https://mediapipe.page.link/blazeface-mc)
|
||||
* [Models and model cards](./models.md#face_detection)
|
||||
|
||||
+240
-19
@@ -19,13 +19,18 @@ landmarks in real-time even on mobile devices. It employs machine learning (ML)
|
||||
to infer the 3D surface geometry, requiring only a single camera input without
|
||||
the need for a dedicated depth sensor. Utilizing lightweight model architectures
|
||||
together with GPU acceleration throughout the pipeline, the solution delivers
|
||||
real-time performance critical for live experiences. The core of the solution is
|
||||
the same as what powers
|
||||
[YouTube Stories](https://youtube-creators.googleblog.com/2018/11/introducing-more-ways-to-share-your.html)'
|
||||
creator effects, the
|
||||
[Augmented Faces API in ARCore](https://developers.google.com/ar/develop/java/augmented-faces/)
|
||||
and the
|
||||
[ML Kit Face Contour Detection API](https://firebase.google.com/docs/ml-kit/face-detection-concepts#contours).
|
||||
real-time performance critical for live experiences.
|
||||
|
||||
Additionally, the solution is bundled with the Face Geometry module that bridges
|
||||
the gap between the face landmark estimation and useful real-time augmented
|
||||
reality (AR) applications. It establishes a metric 3D space and uses the face
|
||||
landmark screen positions to estimate face geometry within that space. The face
|
||||
geometry data consists of common 3D geometry primitives, including a face pose
|
||||
transformation matrix and a triangular face mesh. Under the hood, a lightweight
|
||||
statistical analysis method called
|
||||
[Procrustes Analysis](https://en.wikipedia.org/wiki/Procrustes_analysis) is
|
||||
employed to drive a robust, performant and portable logic. The analysis runs on
|
||||
CPU and has a minimal speed/memory footprint on top of the ML model inference.
|
||||
|
||||
 |
|
||||
:-------------------------------------------------------------: |
|
||||
@@ -67,15 +72,15 @@ Note: To visualize a graph, copy the graph and paste it into
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
## Models
|
||||
### Models
|
||||
|
||||
### Face Detection Model
|
||||
#### Face Detection Model
|
||||
|
||||
The face detector is the same [BlazeFace](https://arxiv.org/abs/1907.05047)
|
||||
model used in [MediaPipe Face Detection](./face_detection.md). Please refer to
|
||||
[MediaPipe Face Detection](./face_detection.md) for details.
|
||||
|
||||
### Face Landmark Model
|
||||
#### Face Landmark Model
|
||||
|
||||
For 3D face landmarks we employed transfer learning and trained a network with
|
||||
several objectives: the network simultaneously predicts 3D landmark coordinates
|
||||
@@ -98,7 +103,108 @@ You can find more information about the face landmark model in this
|
||||
|
||||
 |
|
||||
:------------------------------------------------------------------------: |
|
||||
*Fig 2. Output of MediaPipe Face Mesh: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
|
||||
*Fig 2. Face landmarks: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
|
||||
|
||||
## Face Geometry Module
|
||||
|
||||
The [Face Landmark Model](#face-landmark-model) performs a single-camera face landmark
|
||||
detection in the screen coordinate space: the X- and Y- coordinates are
|
||||
normalized screen coordinates, while the Z coordinate is relative and is scaled
|
||||
as the X coodinate under the
|
||||
[weak perspective projection camera model](https://en.wikipedia.org/wiki/3D_projection#Weak_perspective_projection).
|
||||
This format is well-suited for some applications, however it does not directly
|
||||
enable the full spectrum of augmented reality (AR) features like aligning a
|
||||
virtual 3D object with a detected face.
|
||||
|
||||
The
|
||||
[Face Geometry module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry)
|
||||
moves away from the screen coordinate space towards a metric 3D space and
|
||||
provides necessary primitives to handle a detected face as a regular 3D object.
|
||||
By design, you'll be able to use a perspective camera to project the final 3D
|
||||
scene back into the screen coordinate space with a guarantee that the face
|
||||
landmark positions are not changed.
|
||||
|
||||
### Key Concepts
|
||||
|
||||
#### Metric 3D Space
|
||||
|
||||
The **Metric 3D space** established within the Face Geometry module is a
|
||||
right-handed orthonormal metric 3D coordinate space. Within the space, there is
|
||||
a **virtual perspective camera** located at the space origin and pointed in the
|
||||
negative direction of the Z-axis. In the current pipeline, it is assumed that
|
||||
the input camera frames are observed by exactly this virtual camera and
|
||||
therefore its parameters are later used to convert the screen landmark
|
||||
coordinates back into the Metric 3D space. The *virtual camera parameters* can
|
||||
be set freely, however for better results it is advised to set them as close to
|
||||
the *real physical camera parameters* as possible.
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------: |
|
||||
*Fig 3. A visualization of multiple key elements in the Metric 3D space.* |
|
||||
|
||||
#### Canonical Face Model
|
||||
|
||||
The **Canonical Face Model** is a static 3D model of a human face, which follows
|
||||
the 468 3D face landmark topology of the
|
||||
[Face Landmark Model](#face-landmark-model). The model bears two important
|
||||
functions:
|
||||
|
||||
- **Defines metric units**: the scale of the canonical face model defines the
|
||||
metric units of the Metric 3D space. A metric unit used by the
|
||||
[default canonical face model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.fbx)
|
||||
is a centimeter;
|
||||
- **Bridges static and runtime spaces**: the face pose transformation matrix
|
||||
is - in fact - a linear map from the canonical face model into the runtime
|
||||
face landmark set estimated on each frame. This way, virtual 3D assets
|
||||
modeled around the canonical face model can be aligned with a tracked face
|
||||
by applying the face pose transformation matrix to them.
|
||||
|
||||
### Components
|
||||
|
||||
#### Geometry Pipeline
|
||||
|
||||
The **Geometry Pipeline** is a key component, which is responsible for
|
||||
estimating face geometry objects within the Metric 3D space. On each frame, the
|
||||
following steps are executed in the given order:
|
||||
|
||||
- Face landmark screen coordinates are converted into the Metric 3D space
|
||||
coordinates;
|
||||
- Face pose transformation matrix is estimated as a rigid linear mapping from
|
||||
the canonical face metric landmark set into the runtime face metric landmark
|
||||
set in a way that minimizes a difference between the two;
|
||||
- A face mesh is created using the runtime face metric landmarks as the vertex
|
||||
positions (XYZ), while both the vertex texture coordinates (UV) and the
|
||||
triangular topology are inherited from the canonical face model.
|
||||
|
||||
The geometry pipeline is implemented as a MediaPipe
|
||||
[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc).
|
||||
For your convenience, the face geometry pipeline calculator is bundled together
|
||||
with the face landmark module into a unified MediaPipe
|
||||
[subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_front_gpu.pbtxt).
|
||||
The face geometry format is defined as a Protocol Buffer
|
||||
[message](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto).
|
||||
|
||||
#### Effect Renderer
|
||||
|
||||
The **Effect Renderer** is a component, which serves as a working example of a
|
||||
face effect renderer. It targets the *OpenGL ES 2.0* API to enable a real-time
|
||||
performance on mobile devices and supports the following rendering modes:
|
||||
|
||||
- **3D object rendering mode**: a virtual object is aligned with a detected
|
||||
face to emulate an object attached to the face (example: glasses);
|
||||
- **Face mesh rendering mode**: a texture is stretched on top of the face mesh
|
||||
surface to emulate a face painting technique.
|
||||
|
||||
In both rendering modes, the face mesh is first rendered as an occluder straight
|
||||
into the depth buffer. This step helps to create a more believable effect via
|
||||
hiding invisible elements behind the face surface.
|
||||
|
||||
The effect renderer is implemented as a MediaPipe
|
||||
[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/effect_renderer_calculator.cc).
|
||||
|
||||
|  |
|
||||
| :---------------------------------------------------------------------: |
|
||||
| *Fig 4. An example of face effects rendered by the Face Geometry Effect Renderer.* |
|
||||
|
||||
## Example Apps
|
||||
|
||||
@@ -111,7 +217,12 @@ Note: To visualize a graph, copy the graph and paste it into
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
### Mobile
|
||||
### Face Landmark Example
|
||||
|
||||
Face landmark example showcases real-time, cross-platform face landmark
|
||||
detection. For visual reference, please refer to *Fig. 2*.
|
||||
|
||||
#### Mobile
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
|
||||
@@ -127,7 +238,7 @@ it, for Android modify `NUM_FACES` in
|
||||
and for iOS modify `kNumFaces` in
|
||||
[FaceMeshGpuViewController.mm](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.mm).
|
||||
|
||||
### Desktop
|
||||
#### Desktop
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
@@ -143,18 +254,128 @@ and for iOS modify `kNumFaces` in
|
||||
Tip: Maximum number of faces to detect/process is set to 1 by default. To change
|
||||
it, in the graph file modify the option of `ConstantSidePacketCalculator`.
|
||||
|
||||
#### Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described below and in this
|
||||
[colab](https://mediapipe.page.link/face_mesh_py_colab). If you do need to build
|
||||
the Python package from source, see
|
||||
[additional instructions](../getting_started/building_examples.md#python).
|
||||
|
||||
Activate a Python virtual environment:
|
||||
|
||||
```bash
|
||||
$ python3 -m venv mp_env && source mp_env/bin/activate
|
||||
```
|
||||
|
||||
Install MediaPipe Python package:
|
||||
|
||||
```bash
|
||||
(mp_env)$ pip install mediapipe
|
||||
```
|
||||
|
||||
Run the following Python code:
|
||||
|
||||
<!-- Do not change the example code below directly. Change the corresponding example in mediapipe/python/solutions/face_mesh.py and copy it over. -->
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_face_mesh = mp.solutions.face_mesh
|
||||
|
||||
# For static images:
|
||||
face_mesh = mp_face_mesh.FaceMesh(
|
||||
static_image_mode=True,
|
||||
max_num_faces=1,
|
||||
min_detection_confidence=0.5)
|
||||
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Print and draw face mesh landmarks on the image.
|
||||
if not results.multi_face_landmarks:
|
||||
continue
|
||||
annotated_image = image.copy()
|
||||
for face_landmarks in results.multi_face_landmarks:
|
||||
print('face_landmarks:', face_landmarks)
|
||||
mp_drawing.draw_landmarks(
|
||||
image=annotated_image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACE_CONNECTIONS,
|
||||
landmark_drawing_spec=drawing_spec,
|
||||
connection_drawing_spec=drawing_spec)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', image)
|
||||
face_mesh.close()
|
||||
|
||||
# For webcam input:
|
||||
face_mesh = mp_face_mesh.FaceMesh(
|
||||
min_detection_confidence=0.5, min_tracking_confidence=0.5)
|
||||
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = face_mesh.process(image)
|
||||
|
||||
# Draw the face mesh annotations on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.multi_face_landmarks:
|
||||
for face_landmarks in results.multi_face_landmarks:
|
||||
mp_drawing.draw_landmarks(
|
||||
image=image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACE_CONNECTIONS,
|
||||
landmark_drawing_spec=drawing_spec,
|
||||
connection_drawing_spec=drawing_spec)
|
||||
cv2.imshow('MediaPipe FaceMesh', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
face_mesh.close()
|
||||
cap.release()
|
||||
```
|
||||
|
||||
### Face Effect Example
|
||||
|
||||
Face effect example showcases real-time mobile face effect application use case
|
||||
for the Face Mesh solution. To enable a better user experience, this example
|
||||
only works for a single face. For visual reference, please refer to *Fig. 4*.
|
||||
|
||||
#### Mobile
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/face_effect/face_effect_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_effect/face_effect_gpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1ccnaDnffEuIXriBZr2SK_Eu4FpO7K44s)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/faceeffect`](http:/mediapipe/examples/ios/faceeffect/BUILD)
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
[Real-Time AR Self-Expression with Machine Learning](https://ai.googleblog.com/2019/03/real-time-ar-self-expression-with.html)
|
||||
* TensorFlow Blog:
|
||||
[Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
|
||||
* Google Developers Blog:
|
||||
[MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
|
||||
* Paper:
|
||||
[Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724)
|
||||
([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212))
|
||||
* Face detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite)
|
||||
* Face landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
|
||||
* [Model card](https://mediapipe.page.link/facemesh-mc)
|
||||
* Canonical face model:
|
||||
[FBX](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.fbx),
|
||||
[OBJ](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.obj),
|
||||
[UV visualization](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png)
|
||||
* [Models and model cards](./models.md#face_mesh)
|
||||
|
||||
@@ -54,5 +54,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
[Real-time Hair segmentation and recoloring on Mobile GPUs](https://arxiv.org/abs/1907.06740)
|
||||
([presentation](https://drive.google.com/file/d/1C8WYlWdDRNtU1_pYBvkkG5Z5wqYqf0yj/view))
|
||||
([supplementary video](https://drive.google.com/file/d/1LPtM99Ch2ogyXYbDNpEqnUfhFq0TfLuf/view))
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
|
||||
* [Model card](https://mediapipe.page.link/hairsegmentation-mc)
|
||||
* [Models and model cards](./models.md#hair_segmentation)
|
||||
|
||||
+112
-56
@@ -55,13 +55,21 @@ frame, and only when the landmark model could no longer identify hand presence
|
||||
is palm detection invoked to relocalize the hand.
|
||||
|
||||
The pipeline is implemented as a MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt),
|
||||
which internally utilizes a
|
||||
[palm/hand detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_detection_gpu.pbtxt),
|
||||
a
|
||||
[hand landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt)
|
||||
and a
|
||||
[renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt).
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt)
|
||||
that uses a
|
||||
[hand landmark tracking subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu.pbtxt)
|
||||
from the
|
||||
[hand landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark),
|
||||
and renders using a dedicated
|
||||
[hand renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_renderer_gpu.pbtxt).
|
||||
The
|
||||
[hand landmark tracking subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu.pbtxt)
|
||||
internally uses a
|
||||
[hand landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_gpu.pbtxt)
|
||||
from the same module and a
|
||||
[palm detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_gpu.pbtxt)
|
||||
from the
|
||||
[palm detection module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection).
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
@@ -146,34 +154,11 @@ to visualize its associated subgraphs, please see
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handtrackinggpu/BUILD)
|
||||
|
||||
#### With Multi-hand Support
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1Wk6V9EVaz1ks_MInPqqVGvvJD01SGXDc)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu:multihandtrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/multihandtrackinggpu/BUILD)
|
||||
|
||||
There are two key differences between this graph and that in the
|
||||
[main example](#main-example) (which handles only one hand):
|
||||
|
||||
1. There is a `NormalizedRectVectorHasMinSize` calculator, that checks if in
|
||||
input vector of `NormalizedRect` objects has a minimum size equal to `N`. In
|
||||
this graph, if the vector contains fewer than `N` objects,
|
||||
`MultiHandDetection` subgraph runs. Otherwise, the `GateCalculator` doesn't
|
||||
send any image packets to the `MultiHandDetection` subgraph. This way, the
|
||||
main graph is efficient in that it avoids running the costly hand detection
|
||||
step when there are already `N` hands in the frame.
|
||||
2. The `MergeCalculator` has been replaced by the `AssociationNormRect`
|
||||
calculator. This `AssociationNormRect` takes as input a vector of
|
||||
`NormalizedRect` objects from the `MultiHandDetection` subgraph on the
|
||||
current frame, and a vector of `NormalizedRect` objects from the
|
||||
`MultiHandLandmark` subgraph from the previous frame, and performs an
|
||||
association operation between these objects. This calculator ensures that
|
||||
the output vector doesn't contain overlapping regions based on the specified
|
||||
`min_similarity_threshold`.
|
||||
Tip: Maximum number of hands to detect/process is set to 2 by default. To change
|
||||
it, for Android modify `NUM_HANDS` in
|
||||
[MainActivity.java](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java),
|
||||
and for iOS modify `kNumHands` in
|
||||
[HandTrackingViewController.mm](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handtrackinggpu/HandTrackingViewController.mm).
|
||||
|
||||
#### Palm/Hand Detection Only (no landmarks)
|
||||
|
||||
@@ -187,8 +172,6 @@ There are two key differences between this graph and that in the
|
||||
|
||||
### Desktop
|
||||
|
||||
#### Main Example
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt)
|
||||
@@ -196,22 +179,101 @@ There are two key differences between this graph and that in the
|
||||
[`mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/hand_tracking/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt)
|
||||
[`mediapipe/graphs/hand_tracking/hand_tracking_desktop_live_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_desktop_gpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/hand_tracking/BUILD)
|
||||
|
||||
#### With Multi-hand Support
|
||||
Tip: Maximum number of hands to detect/process is set to 2 by default. To change
|
||||
it, in the graph file modify the option of `ConstantSidePacketCalculator`.
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/multi_hand_tracking/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/multi_hand_tracking/BUILD)
|
||||
### Python
|
||||
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described below and in this
|
||||
[colab](https://mediapipe.page.link/hands_py_colab). If you do need to build the
|
||||
Python package from source, see
|
||||
[additional instructions](../getting_started/building_examples.md#python).
|
||||
|
||||
Activate a Python virtual environment:
|
||||
|
||||
```bash
|
||||
$ python3 -m venv mp_env && source mp_env/bin/activate
|
||||
```
|
||||
|
||||
Install MediaPipe Python package:
|
||||
|
||||
```bash
|
||||
(mp_env)$ pip install mediapipe
|
||||
```
|
||||
|
||||
Run the following Python code:
|
||||
|
||||
<!-- Do not change the example code below directly. Change the corresponding example in mediapipe/python/solutions/hands.py and copy it over. -->
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_hands = mp.solutions.hands
|
||||
|
||||
# For static images:
|
||||
hands = mp_hands.Hands(
|
||||
static_image_mode=True,
|
||||
max_num_hands=2,
|
||||
min_detection_confidence=0.7)
|
||||
for idx, file in enumerate(file_list):
|
||||
# Read an image, flip it around y-axis for correct handedness output (see
|
||||
# above).
|
||||
image = cv2.flip(cv2.imread(file), 1)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Print handedness and draw hand landmarks on the image.
|
||||
print('handedness:', results.multi_handedness)
|
||||
if not results.multi_hand_landmarks:
|
||||
continue
|
||||
annotated_image = image.copy()
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
print('hand_landmarks:', hand_landmarks)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
cv2.imwrite(
|
||||
'/tmp/annotated_image' + str(idx) + '.png', cv2.flip(image, 1))
|
||||
hands.close()
|
||||
|
||||
# For webcam input:
|
||||
hands = mp_hands.Hands(
|
||||
min_detection_confidence=0.7, min_tracking_confidence=0.5)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = hands.process(image)
|
||||
|
||||
# Draw the hand annotations on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.multi_hand_landmarks:
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
mp_drawing.draw_landmarks(
|
||||
image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Hands', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
hands.close()
|
||||
cap.release()
|
||||
```
|
||||
|
||||
Tip: Use command `deactivate` to exit the Python virtual environment.
|
||||
|
||||
### Web
|
||||
|
||||
@@ -226,10 +288,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
* Paper:
|
||||
[MediaPipe Hands: On-device Real-time Hand Tracking](https://arxiv.org/abs/2006.10214)
|
||||
([presentation](https://www.youtube.com/watch?v=I-UOrvxxXEk))
|
||||
* Palm detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
|
||||
* Hand landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
|
||||
* [Model card](https://mediapipe.page.link/handmc)
|
||||
* [Models and model cards](./models.md#hands)
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
---
|
||||
layout: default
|
||||
title: Instant Motion Tracking
|
||||
parent: Solutions
|
||||
nav_order: 9
|
||||
---
|
||||
|
||||
# MediaPipe Instant Motion Tracking
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Augmented Reality (AR) technology creates fun, engaging, and immersive user
|
||||
experiences. The ability to perform AR tracking across devices and platforms,
|
||||
without initialization, remains important to power AR applications at scale.
|
||||
|
||||
MediaPipe Instant Motion Tracking provides AR tracking across devices and
|
||||
platforms without initialization or calibration. It is built upon the
|
||||
[MediaPipe Box Tracking](./box_tracking.md) solution. With Instant Motion
|
||||
Tracking, you can easily place virtual 2D and 3D content on static or moving
|
||||
surfaces, allowing them to seamlessly interact with the real-world environment.
|
||||
|
||||
 |
|
||||
:-----------------------------------------------------------------------: |
|
||||
*Fig 1. Instant Motion Tracking is used to augment the world with a 3D sticker.* |
|
||||
|
||||
## Pipeline
|
||||
|
||||
The Instant Motion Tracking pipeline is implemented as a MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt),
|
||||
which internally utilizes a
|
||||
[RegionTrackingSubgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/subgraphs/region_tracking.pbtxt)
|
||||
in order to perform anchor tracking for each individual 3D sticker.
|
||||
|
||||
We first use a
|
||||
[StickerManagerCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/calculators/sticker_manager_calculator.cc)
|
||||
to prepare the individual sticker data for the rest of the application. This
|
||||
information is then sent to the
|
||||
[RegionTrackingSubgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/subgraphs/region_tracking.pbtxt)
|
||||
that performs 3D region tracking for sticker placement and rendering. Once
|
||||
acquired, our tracked sticker regions are sent with user transformations (i.e.
|
||||
gestures from the user to rotate and zoom the sticker) and IMU data to the
|
||||
[MatricesManagerCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc),
|
||||
which turns all our sticker transformation data into a set of model matrices.
|
||||
This data is handled directly by our
|
||||
[GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc)
|
||||
as an input stream, which will render the provided texture and object file using
|
||||
our matrix specifications. The output of
|
||||
[GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc)
|
||||
is a video stream depicting the virtual 3D content rendered on top of the real
|
||||
world, creating immersive AR experiences for users.
|
||||
|
||||
## Using Instant Motion Tracking
|
||||
|
||||
With the Instant Motion Tracking MediaPipe [graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt),
|
||||
an application can create an interactive and realistic AR experience by
|
||||
specifying the required input streams, side packets, and output streams.
|
||||
The input streams are the following:
|
||||
|
||||
* Input Video (GpuBuffer): Video frames to render augmented stickers onto.
|
||||
* Rotation Matrix (9-element Float Array): The 3x3 row-major rotation
|
||||
matrix from the device IMU to determine proper orientation of the device.
|
||||
* Sticker Proto String (String): A string representing the
|
||||
serialized [sticker buffer protobuf message](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/calculators/sticker_buffer.proto),
|
||||
containing a list of all stickers and their attributes.
|
||||
* Each sticker in the Protobuffer has a unique ID to find associated
|
||||
anchors and transforms, an initial anchor placement in a normalized [0.0, 1.0]
|
||||
3D space, a user rotation and user scaling transform on the sticker,
|
||||
and an integer indicating which type of objects to render for the
|
||||
sticker (e.g. 3D asset or GIF).
|
||||
* Sticker Sentinel (Integer): When an anchor must be initially placed or
|
||||
repositioned, this value must be changed to the ID of the anchor to reset from
|
||||
the sticker buffer protobuf message. If no valid ID is provided, the system
|
||||
will simply maintain tracking.
|
||||
|
||||
Side packets are also an integral part of the Instant Motion Tracking solution
|
||||
to provide device-specific information for the rendering system:
|
||||
|
||||
* Field of View (Float): The field of view of the camera in radians.
|
||||
* Aspect Ratio (Float): The aspect ratio (width / height) of the camera frames
|
||||
(this ratio corresponds to the image frames themselves, not necessarily the
|
||||
screen bounds).
|
||||
* Object Asset (String): The
|
||||
[GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc)
|
||||
must be provided with an associated asset file name pointing to the 3D model
|
||||
to render in the viewfinder.
|
||||
* (Optional) Texture (ImageFrame on Android, GpuBuffer on iOS): Textures for
|
||||
the
|
||||
[GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc)
|
||||
can be provided either via an input stream (dynamic texturing) or as a side
|
||||
packet (unchanging texture).
|
||||
|
||||
The rendering system for the Instant Motion Tracking is powered by OpenGL. For
|
||||
more information regarding the structure of model matrices and OpenGL rendering,
|
||||
please visit [OpenGL Wiki](https://www.khronos.org/opengl/wiki/). With the
|
||||
specifications above, the Instant Motion Tracking capabilities can be adapted to
|
||||
any device that is able to run the MediaPipe framework with a working IMU system
|
||||
and connected camera.
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/building_examples.md#android) on how to build
|
||||
MediaPipe examples.
|
||||
|
||||
* Graph: [mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt)
|
||||
|
||||
* Android target (or download prebuilt [ARM64 APK](https://drive.google.com/file/d/1KnaBBoKpCHR73nOBJ4fL_YdWVTAcwe6L/view?usp=sharing)):
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking:instantmotiontracking`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD)
|
||||
|
||||
## Resources
|
||||
|
||||
* Google Developers Blog:
|
||||
[Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
|
||||
* Google AI Blog:
|
||||
[The Instant Motion Tracking Behind Motion Stills AR](https://ai.googleblog.com/2018/02/the-instant-motion-tracking-behind.html)
|
||||
* Paper:
|
||||
[Instant Motion Tracking and Its Applications to Augmented Reality](https://arxiv.org/abs/1907.06796)
|
||||
@@ -55,7 +55,7 @@ that uses a
|
||||
from the
|
||||
[face landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark),
|
||||
an
|
||||
[iris landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_tracking/iris_landmark_left_and_right_gpu.pbtxt)
|
||||
[iris landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark_left_and_right_gpu.pbtxt)
|
||||
from the
|
||||
[iris landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark),
|
||||
and renders using a dedicated
|
||||
@@ -72,6 +72,11 @@ Note: To visualize a graph, copy the graph and paste it into
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
The output of the pipeline is a set of 478 3D landmarks, including 468 face
|
||||
landmarks from [MediaPipe Face Mesh](./face_mesh.md), with those around the eyes
|
||||
further refined (see Fig 2), and 10 additional iris landmarks appended at the
|
||||
end (5 for each eye, and see Fig 2 also).
|
||||
|
||||
## Models
|
||||
|
||||
### Face Detection Model
|
||||
@@ -194,11 +199,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
* Paper:
|
||||
[Real-time Pupil Tracking from Monocular Video for Digital Puppetry](https://arxiv.org/abs/2006.11341)
|
||||
([presentation](https://youtu.be/cIhXkiiapQI))
|
||||
* Face detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite)
|
||||
* Face landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
|
||||
* Iris landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark.tflite)
|
||||
* [Model card](https://mediapipe.page.link/iris-mc)
|
||||
* [Models and model cards](./models.md#iris)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: KNIFT (Template-based Feature Matching)
|
||||
parent: Solutions
|
||||
nav_order: 10
|
||||
nav_order: 11
|
||||
---
|
||||
|
||||
# MediaPipe KNIFT
|
||||
@@ -139,7 +139,4 @@ to run regular TFLite inference.
|
||||
|
||||
* Google Developers Blog:
|
||||
[MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
|
||||
* [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
|
||||
* [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
|
||||
* [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
|
||||
* [Model card](https://mediapipe.page.link/knift-mc)
|
||||
* [Models and model cards](./models.md#knift)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: Dataset Preparation with MediaSequence
|
||||
parent: Solutions
|
||||
nav_order: 12
|
||||
nav_order: 13
|
||||
---
|
||||
|
||||
# Dataset Preparation with MediaSequence
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
---
|
||||
layout: default
|
||||
title: Models and Model Cards
|
||||
parent: Solutions
|
||||
nav_order: 30
|
||||
---
|
||||
|
||||
# MediaPipe Models and Model Cards
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
|
||||
|
||||
* Face detection model for front-facing/selfie camera:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
|
||||
[TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
|
||||
* Face detection model for back-facing camera:
|
||||
[TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
|
||||
* [Model card](https://mediapipe.page.link/blazeface-mc)
|
||||
|
||||
### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)
|
||||
|
||||
* Face landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
|
||||
* [Model card](https://mediapipe.page.link/facemesh-mc)
|
||||
|
||||
### [Iris](https://google.github.io/mediapipe/solutions/iris)
|
||||
|
||||
* Iris landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark.tflite)
|
||||
* [Model card](https://mediapipe.page.link/iris-mc)
|
||||
|
||||
### [Hands](https://google.github.io/mediapipe/solutions/hands)
|
||||
|
||||
* Palm detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
|
||||
* Hand landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
|
||||
* [Model card](https://mediapipe.page.link/handmc)
|
||||
|
||||
### [Pose](https://google.github.io/mediapipe/solutions/pose)
|
||||
|
||||
* Pose detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
|
||||
* Upper-body pose landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
|
||||
* [Model card](https://mediapipe.page.link/blazepose-mc)
|
||||
|
||||
### [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation)
|
||||
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
|
||||
* [Model card](https://mediapipe.page.link/hairsegmentation-mc)
|
||||
|
||||
### [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)
|
||||
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
|
||||
* [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
|
||||
* [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
|
||||
* [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
|
||||
|
||||
### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
|
||||
|
||||
* [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
|
||||
* [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
|
||||
* [TFLite model for cameras](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_camera.tflite)
|
||||
* [TFLite model for cups](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_cup.tflite)
|
||||
* [Single-stage TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers_1stage.tflite)
|
||||
* [Single-stage TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair_1stage.tflite)
|
||||
* [Model card](https://mediapipe.page.link/objectron-mc)
|
||||
|
||||
### [KNIFT](https://google.github.io/mediapipe/solutions/knift)
|
||||
|
||||
* [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
|
||||
* [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
|
||||
* [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
|
||||
* [Model card](https://mediapipe.page.link/knift-mc)
|
||||
@@ -144,7 +144,4 @@ to cross-compile and run MediaPipe examples on the
|
||||
|
||||
## Resources
|
||||
|
||||
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
|
||||
* [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
|
||||
* [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
|
||||
* [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
|
||||
* [Models and model cards](./models.md#object_detection)
|
||||
|
||||
+100
-30
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: Objectron (3D Object Detection)
|
||||
parent: Solutions
|
||||
nav_order: 9
|
||||
nav_order: 10
|
||||
---
|
||||
|
||||
# MediaPipe Objectron
|
||||
@@ -15,13 +15,12 @@ nav_order: 9
|
||||
## Overview
|
||||
|
||||
MediaPipe Objectron is a mobile real-time 3D object detection solution for
|
||||
everyday objects. It detects objects in 2D images, and estimates their poses and
|
||||
sizes through a machine learning (ML) model, trained on a newly created 3D
|
||||
dataset.
|
||||
everyday objects. It detects objects in 2D images, and estimates their poses
|
||||
through a machine learning (ML) model, trained on a newly created 3D dataset.
|
||||
|
||||
 | 
|
||||
:--------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------:
|
||||
*Fig 1(a). Objectron for Shoes.* | *Fig 1(b). Objectron for Chairs.*
|
||||
 |  |  | 
|
||||
:--------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------: | :------------------------------------------------------------------------------------: | :------------------------------------------------------------------------------:
|
||||
*Fig 1(a). Objectron for Shoes.* | *Fig 1(b). Objectron for Chairs.* | *Fig 1(c). Objectron for Cameras.* | *Fig 1(d). Objectron for Cups.*
|
||||
|
||||
Object detection is an extensively studied computer vision problem, but most of
|
||||
the research has focused on
|
||||
@@ -85,15 +84,41 @@ able to increase the accuracy by about 10%.
|
||||
:-------------------------------------------------------------------------------------------: |
|
||||
*Fig 4. An example of AR synthetic data generation. The virtual white-brown cereal box is rendered into the real scene, next to the real blue book.* |
|
||||
|
||||
## ML Model for 3D Object Detection
|
||||
## ML Pipelines for 3D Object Detection
|
||||
|
||||
We built two ML pipelines to predict the 3D bounding box of an object from a
|
||||
single RGB image: one is a two-stage pipeline and the other is a single-stage
|
||||
pipeline. The two-stage pipeline is 3x faster than the single-stage pipeline
|
||||
with similar or better accuracy. The single stage pipeline is good at detecting
|
||||
multiple objects, whereas the two stage pipeline is good for a single dominant
|
||||
object.
|
||||
|
||||
### Two-stage Pipeline
|
||||
|
||||
Our two-stage pipeline is illustrated by the diagram in Fig 5. The first stage
|
||||
uses an object detector to find the 2D crop of the object. The second stage
|
||||
takes the image crop and estimates the 3D bounding box. At the same time, it
|
||||
also computes the 2D crop of the object for the next frame, such that the object
|
||||
detector does not need to run every frame.
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------------------: |
|
||||
*Fig 5. Network architecture and post-processing for two-stage 3D object detection.* |
|
||||
|
||||
We can use any 2D object detector for the first stage. In this solution, we use
|
||||
[TensorFlow Object Detection](https://github.com/tensorflow/models/tree/master/research/object_detection).
|
||||
The second stage 3D bounding box predictor we released runs 83FPS on Adreno 650
|
||||
mobile GPU.
|
||||
|
||||
### Single-stage Pipeline
|
||||
|
||||
 |
|
||||
:---------------------------------------------------------------------------------: |
|
||||
*Fig 5. Network architecture and post-processing for 3D object detection.* |
|
||||
*Fig 6. Network architecture and post-processing for single-stage 3D object detection.* |
|
||||
|
||||
We [built a single-stage model](https://arxiv.org/abs/2003.03522) to predict the
|
||||
pose and physical size of an object from a single RGB image. The model backbone
|
||||
has an encoder-decoder architecture, built upon
|
||||
Our [single-stage pipeline](https://arxiv.org/abs/2003.03522) is illustrated by
|
||||
the diagram in Fig 6, the model backbone has an encoder-decoder architecture,
|
||||
built upon
|
||||
[MobileNetv2](https://ai.googleblog.com/2018/04/mobilenetv2-next-generation-of-on.html).
|
||||
We employ a multi-task learning approach, jointly predicting an object's shape
|
||||
with detection and regression. The shape task predicts the object's shape
|
||||
@@ -114,9 +139,9 @@ size of the object. The model is light enough to run real-time on mobile devices
|
||||
|
||||
 |
|
||||
:-------------------------------------------------------------------------------------: |
|
||||
*Fig 6. Sample results of our network — (Left) original 2D image with estimated bounding boxes, (Middle) object detection by Gaussian distribution, (Right) predicted segmentation mask.* |
|
||||
*Fig 7. Sample results of our network — (Left) original 2D image with estimated bounding boxes, (Middle) object detection by Gaussian distribution, (Right) predicted segmentation mask.* |
|
||||
|
||||
## Detection and Tracking Pipeline
|
||||
#### Detection and Tracking
|
||||
|
||||
When the model is applied to every frame captured by the mobile device, it can
|
||||
suffer from jitter due to the ambiguity of the 3D bounding box estimated in each
|
||||
@@ -130,7 +155,7 @@ temporally consistent, reducing the jitter.
|
||||
|
||||
The Objectron 3D object detection and tracking pipeline is implemented as a
|
||||
MediaPipe
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt),
|
||||
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt),
|
||||
which internally uses a
|
||||
[detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/subgraphs/objectron_detection_gpu.pbtxt)
|
||||
and a
|
||||
@@ -147,6 +172,12 @@ new detection becomes available from the detection subgraph, the tracking
|
||||
subgraph is also responsible for consolidation between the detection and
|
||||
tracking results, based on the area of overlap.
|
||||
|
||||
## Objectron Dataset
|
||||
|
||||
We also released our [Objectron dataset](http://objectron.dev), with which we
|
||||
trained our 3D object detection models. The technical details of the Objectron
|
||||
dataset, including usage and tutorials, are available on the dataset website.
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
@@ -158,32 +189,72 @@ Note: To visualize a graph, copy the graph and paste it into
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
### Objectron for Shoes
|
||||
### Two-stage Objectron
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1S0K4hbWt3o31FfQ4QU3Rz7IHrvOUMx1d)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD)
|
||||
* iOS target: Not available
|
||||
[`mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
|
||||
|
||||
### Objectron for Chairs
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/chair_classic_occlusion_tracking.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/open?id=1MM8K-13bXLCVS1EHQ-KgkVyEahEPrKej)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD)
|
||||
and add `--define chair=true` to the build command, i.e.,
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD).
|
||||
|
||||
Build for **shoes** (default) with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1ANW9WDOCb8QO1r8gDC03A4UgrPkICdPP/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Build for **chairs** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1lcUv1TBnv_SxnKSQwdOqbdLa9mkaTJHy/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define chair=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Build for **cups** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1bf77KDkowwrduleiC9B1M1XnEhjnOQbX/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define cup=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Build for **cameras** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1GM7lPO-s5URVxIzQur1bLsionEJs3yIl/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define camera=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
### Single-stage Objectron
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
|
||||
|
||||
* Android target:
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD).
|
||||
|
||||
Build with **single-stage** model for **shoes** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1MvaEg4dkvKN8jAU1Z2GtudyXi1rQHYsE/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define shoe_1stage=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Build with **single-stage** model for **chairs** with:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1GJL4z3jr-wD1jMHGd4NBfOG-Yoq5t167/view?usp=sharing)
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 --define chair_1stage=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
[Announcing the Objectron Dataset](https://mediapipe.page.link/objectron_dataset_ai_blog)
|
||||
* Google AI Blog:
|
||||
[Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
|
||||
* Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
|
||||
@@ -191,5 +262,4 @@ to visualize its associated subgraphs, please see
|
||||
* Paper:
|
||||
[Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
|
||||
([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
|
||||
* [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
|
||||
* [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
|
||||
* [Models and model cards](./models.md#objectron)
|
||||
|
||||
+90
-34
@@ -22,12 +22,13 @@ on top of the physical world in augmented reality.
|
||||
|
||||
MediaPipe Pose is a ML solution for high-fidelity upper-body pose tracking,
|
||||
inferring 25 2D upper-body landmarks from RGB video frames utilizing our
|
||||
[BlazePose](https://mediapipe.page.link/blazepose-blog) research. Current
|
||||
state-of-the-art approaches rely primarily on powerful desktop environments for
|
||||
inference, whereas our method achieves real-time performance on most modern
|
||||
[mobile phones](#mobile), [desktops/laptops](#desktop), in [python](#python) and
|
||||
even on the [web](#web). A variant of MediaPipe Pose that performs full-body
|
||||
pose tracking on mobile phones will be included in an upcoming release of
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
research. Current state-of-the-art approaches rely primarily on powerful desktop
|
||||
environments for inference, whereas our method achieves real-time performance on
|
||||
most modern [mobile phones](#mobile), [desktops/laptops](#desktop), in
|
||||
[python](#python) and even on the [web](#web). A variant of MediaPipe Pose that
|
||||
performs full-body pose tracking on mobile phones will be included in an
|
||||
upcoming release of
|
||||
[ML Kit](https://developers.google.com/ml-kit/early-access/pose-detection).
|
||||
|
||||
 |
|
||||
@@ -87,12 +88,13 @@ hip midpoints.
|
||||
### Pose Landmark Model (BlazePose Tracker)
|
||||
|
||||
The landmark model currently included in MediaPipe Pose predicts the location of
|
||||
25 upper-body landmarks (see figure below), with three degrees of freedom each
|
||||
(x, y location and visibility), plus two virtual alignment keypoints. It shares
|
||||
the same architecture as the full-body version that predicts 33 landmarks,
|
||||
described in more detail in the
|
||||
[BlazePose Google AI Blog](https://mediapipe.page.link/blazepose-blog) and in
|
||||
this [paper](https://arxiv.org/abs/2006.10204).
|
||||
25 upper-body landmarks (see figure below), each with `(x, y, z, visibility)`.
|
||||
Note that the `z` value should be discarded as the model is currently not fully
|
||||
trained to predict depth, but this is something we have on the roadmap. The
|
||||
model shares the same architecture as the full-body version that predicts 33
|
||||
landmarks, described in more detail in the
|
||||
[BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
and in this [paper](https://arxiv.org/abs/2006.10204).
|
||||
|
||||
 |
|
||||
:------------------------------------------------------------------------------------------------: |
|
||||
@@ -141,26 +143,84 @@ MediaPipe examples.
|
||||
|
||||
### Python
|
||||
|
||||
Please first see general instructions for
|
||||
[Python](../getting_started/building_examples.md#python) examples.
|
||||
MediaPipe Python package is available on
|
||||
[PyPI](https://pypi.org/project/mediapipe/), and can be installed simply by `pip
|
||||
install mediapipe` on Linux and macOS, as described below and in this
|
||||
[colab](https://mediapipe.page.link/pose_py_colab). If you do need to build the
|
||||
Python package from source, see
|
||||
[additional instructions](../getting_started/building_examples.md#python).
|
||||
|
||||
Activate a Python virtual environment:
|
||||
|
||||
```bash
|
||||
(mp_env)$ python3
|
||||
>>> import mediapipe as mp
|
||||
>>> pose_tracker = mp.examples.UpperBodyPoseTracker()
|
||||
|
||||
# For image input
|
||||
>>> pose_landmarks, _ = pose_tracker.run(input_file='/path/to/input/file', output_file='/path/to/output/file')
|
||||
>>> pose_landmarks, annotated_image = pose_tracker.run(input_file='/path/to/file')
|
||||
|
||||
# For live camera input
|
||||
# (Press Esc within the output image window to stop the run or let it self terminate after 30 seconds.)
|
||||
>>> pose_tracker.run_live()
|
||||
|
||||
# Close the tracker.
|
||||
>>> pose_tracker.close()
|
||||
$ python3 -m venv mp_env && source mp_env/bin/activate
|
||||
```
|
||||
|
||||
Install MediaPipe Python package:
|
||||
|
||||
```bash
|
||||
(mp_env)$ pip install mediapipe
|
||||
```
|
||||
|
||||
Run the following Python code:
|
||||
|
||||
<!-- Do not change the example code below directly. Change the corresponding example in mediapipe/python/solutions/pose.py and copy it over. -->
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_pose = mp.solutions.pose
|
||||
|
||||
# For static images:
|
||||
pose = mp_pose.Pose(
|
||||
static_image_mode=True, min_detection_confidence=0.5)
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Print and draw pose landmarks on the image.
|
||||
print(
|
||||
'nose landmark:',
|
||||
results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE])
|
||||
annotated_image = image.copy()
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', image)
|
||||
pose.close()
|
||||
|
||||
# For webcam input:
|
||||
pose = mp_pose.Pose(
|
||||
min_detection_confidence=0.5, min_tracking_confidence=0.5)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
break
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = pose.process(image)
|
||||
|
||||
# Draw the pose annotation on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Pose', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
pose.close()
|
||||
cap.release()
|
||||
```
|
||||
|
||||
Tip: Use command `deactivate` to exit the Python virtual environment.
|
||||
|
||||
### Web
|
||||
|
||||
Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
@@ -168,12 +228,8 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
[BlazePose - On-device Real-time Body Pose Tracking](https://mediapipe.page.link/blazepose-blog)
|
||||
[BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
* Paper:
|
||||
[BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
|
||||
([presentation](https://youtu.be/YPpUOTRn5tA))
|
||||
* Pose detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
|
||||
* Upper-body pose landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
|
||||
* [Model card](https://mediapipe.page.link/blazepose-mc)
|
||||
* [Models and model cards](./models.md#pose)
|
||||
|
||||
+20
-15
@@ -16,18 +16,23 @@ has_toc: false
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to ../external_index.md. -->
|
||||
|
||||
[]() | Android | iOS | Desktop | Python | Web | Coral
|
||||
:---------------------------------------------------------------------------- | :-----: | :-: | :-----: | :----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | | |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) 🆕 | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) 🆕 | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
[]() | Android | iOS | Desktop | Python | Web | Coral
|
||||
:---------------------------------------------------------------------------------------- | :-----: | :-: | :-----: | :----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | ✅ |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
layout: default
|
||||
title: YouTube-8M Feature Extraction and Model Inference
|
||||
parent: Solutions
|
||||
nav_order: 13
|
||||
nav_order: 14
|
||||
---
|
||||
|
||||
# YouTube-8M Feature Extraction and Model Inference
|
||||
|
||||
@@ -26,9 +26,10 @@ To enable tracing and profiling of a mediapipe graph:
|
||||
1. The profiling library must be linked to the framework.
|
||||
2. Tracing and profiling must be enabled in the graph configuration.
|
||||
|
||||
The profiling library is linked to the framework by default. If needed,
|
||||
the profiling library can be omitted from the framework using the bazel
|
||||
command line option: `--define MEDIAPIPE_PROFILING=0`.
|
||||
The profiling library is linked to the framework by default for Desktop.
|
||||
If needed, it can be omitted from the framework using the bazel command line
|
||||
option: `--define MEDIAPIPE_PROFILING=0`. For other platforms, you can use the
|
||||
bazel command line option `--define MEDIAPIPE_PROFILING=1` to link it.
|
||||
|
||||
To enable tracing and profiling, the `CalculatorGraphConfig` (in
|
||||
[calculator.proto](https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator.proto))
|
||||
@@ -38,6 +39,7 @@ is a simple setup that turns on tracing and keeps 100 seconds of timing events:
|
||||
```
|
||||
profiler_config {
|
||||
trace_enabled: true
|
||||
enable_profiler: true
|
||||
trace_log_interval_count: 200
|
||||
}
|
||||
```
|
||||
@@ -71,6 +73,9 @@ MediaPipe will emit data into a pre-specified directory:
|
||||
|
||||
You can open the Download Container. Logs will be located in `application
|
||||
container/.xcappdata/AppData/Documents/`
|
||||
If XCode shows empty content for the downloaded container file, you can
|
||||
right click and select 'Show Package Contents' in Finder. Logs
|
||||
will be located in 'AppData/Documents/'
|
||||
|
||||
|
||||
|
||||
@@ -144,10 +149,14 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
```bash
|
||||
profiler_config {
|
||||
trace_enabled: true
|
||||
trace_log_path: "/sdcard/profiles"
|
||||
enable_profiler: true
|
||||
trace_log_path: "/sdcard/profiles/"
|
||||
}
|
||||
```
|
||||
|
||||
Note: The forward slash at the end of the `trace_log_path` is necessary for
|
||||
indicating that `profiles` is a directory (that *should* exist).
|
||||
|
||||
* Download the trace files from the device.
|
||||
|
||||
```bash
|
||||
|
||||
@@ -10,11 +10,11 @@
|
||||
"mediapipe/examples/ios/helloworld/BUILD",
|
||||
"mediapipe/examples/ios/facedetectioncpu/BUILD",
|
||||
"mediapipe/examples/ios/facedetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/faceeffect/BUILD",
|
||||
"mediapipe/examples/ios/facemeshgpu/BUILD",
|
||||
"mediapipe/examples/ios/handdetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/handtrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/iristrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/objectdetectioncpu/BUILD",
|
||||
"mediapipe/examples/ios/objectdetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD"
|
||||
@@ -23,11 +23,11 @@
|
||||
"//mediapipe/examples/ios/helloworld:HelloWorldApp",
|
||||
"//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp",
|
||||
"//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/faceeffect:FaceEffectApp",
|
||||
"//mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp",
|
||||
"//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/iristrackinggpu:IrisTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp",
|
||||
"//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp",
|
||||
@@ -90,10 +90,11 @@
|
||||
"mediapipe/examples/ios/helloworld",
|
||||
"mediapipe/examples/ios/facedetectioncpu",
|
||||
"mediapipe/examples/ios/facedetectiongpu",
|
||||
"mediapipe/examples/ios/faceeffect",
|
||||
"mediapipe/examples/ios/faceeffect/Base.lproj",
|
||||
"mediapipe/examples/ios/handdetectiongpu",
|
||||
"mediapipe/examples/ios/handtrackinggpu",
|
||||
"mediapipe/examples/ios/iristrackinggpu",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu",
|
||||
"mediapipe/examples/ios/objectdetectioncpu",
|
||||
"mediapipe/examples/ios/objectdetectiongpu",
|
||||
"mediapipe/examples/ios/upperbodyposetrackinggpu",
|
||||
@@ -110,6 +111,7 @@
|
||||
"mediapipe/graphs",
|
||||
"mediapipe/graphs/edge_detection",
|
||||
"mediapipe/graphs/face_detection",
|
||||
"mediapipe/graphs/face_geometry",
|
||||
"mediapipe/graphs/hand_tracking",
|
||||
"mediapipe/graphs/object_detection",
|
||||
"mediapipe/graphs/pose_tracking",
|
||||
|
||||
@@ -13,11 +13,11 @@
|
||||
"mediapipe/examples/ios",
|
||||
"mediapipe/examples/ios/facedetectioncpu",
|
||||
"mediapipe/examples/ios/facedetectiongpu",
|
||||
"mediapipe/examples/ios/faceeffect",
|
||||
"mediapipe/examples/ios/facemeshgpu",
|
||||
"mediapipe/examples/ios/handdetectiongpu",
|
||||
"mediapipe/examples/ios/handtrackinggpu",
|
||||
"mediapipe/examples/ios/iristrackinggpu",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu",
|
||||
"mediapipe/examples/ios/objectdetectioncpu",
|
||||
"mediapipe/examples/ios/objectdetectiongpu",
|
||||
"mediapipe/examples/ios/upperbodyposetrackinggpu"
|
||||
|
||||
@@ -12,7 +12,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
import mediapipe.examples.python as examples
|
||||
from mediapipe.python import *
|
||||
import mediapipe.util as util
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
filegroup(
|
||||
name = "test_audios",
|
||||
|
||||
@@ -13,181 +13,121 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "concatenate_vector_calculator_proto",
|
||||
srcs = ["concatenate_vector_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "dequantize_byte_array_calculator_proto",
|
||||
srcs = ["dequantize_byte_array_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "packet_cloner_calculator_proto",
|
||||
srcs = ["packet_cloner_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "packet_resampler_calculator_proto",
|
||||
srcs = ["packet_resampler_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "packet_thinner_calculator_proto",
|
||||
srcs = ["packet_thinner_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "split_vector_calculator_proto",
|
||||
srcs = ["split_vector_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "quantize_float_vector_calculator_proto",
|
||||
srcs = ["quantize_float_vector_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "sequence_shift_calculator_proto",
|
||||
srcs = ["sequence_shift_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "gate_calculator_proto",
|
||||
srcs = ["gate_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "constant_side_packet_calculator_proto",
|
||||
srcs = ["constant_side_packet_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/framework/formats:classification_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "clip_vector_size_calculator_proto",
|
||||
srcs = ["clip_vector_size_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "packet_cloner_calculator_cc_proto",
|
||||
srcs = ["packet_cloner_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":packet_cloner_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "packet_resampler_calculator_cc_proto",
|
||||
srcs = ["packet_resampler_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":packet_resampler_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "packet_thinner_calculator_cc_proto",
|
||||
srcs = ["packet_thinner_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":packet_thinner_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "split_vector_calculator_cc_proto",
|
||||
srcs = ["split_vector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":split_vector_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "concatenate_vector_calculator_cc_proto",
|
||||
srcs = ["concatenate_vector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":concatenate_vector_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "clip_vector_size_calculator_cc_proto",
|
||||
srcs = ["clip_vector_size_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":clip_vector_size_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "dequantize_byte_array_calculator_cc_proto",
|
||||
srcs = ["dequantize_byte_array_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":dequantize_byte_array_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "quantize_float_vector_calculator_cc_proto",
|
||||
srcs = ["quantize_float_vector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":quantize_float_vector_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "sequence_shift_calculator_cc_proto",
|
||||
srcs = ["sequence_shift_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":sequence_shift_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "gate_calculator_cc_proto",
|
||||
srcs = ["gate_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":gate_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "constant_side_packet_calculator_cc_proto",
|
||||
srcs = ["constant_side_packet_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":constant_side_packet_calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
@@ -286,11 +226,23 @@ cc_library(
|
||||
name = "concatenate_vector_calculator",
|
||||
srcs = ["concatenate_vector_calculator.cc"],
|
||||
hdrs = ["concatenate_vector_calculator.h"],
|
||||
copts = select({
|
||||
# Needed for "//mediapipe/framework/formats:tensor" compatibility on Apple
|
||||
# platforms for Metal pulled in via the tensor.h header.
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":concatenate_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
@@ -850,14 +802,23 @@ cc_library(
|
||||
name = "split_vector_calculator",
|
||||
srcs = ["split_vector_calculator.cc"],
|
||||
hdrs = ["split_vector_calculator.h"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":split_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:resource_util",
|
||||
@@ -1119,6 +1080,8 @@ cc_library(
|
||||
":constant_side_packet_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message ClipVectorSizeCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional ClipVectorSizeCalculatorOptions ext = 274674998;
|
||||
|
||||
@@ -16,7 +16,10 @@
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
@@ -45,6 +48,12 @@ REGISTER_CALCULATOR(ConcatenateFloatVectorCalculator);
|
||||
typedef ConcatenateVectorCalculator<int32> ConcatenateInt32VectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<uint64> ConcatenateUInt64VectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateUInt64VectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateBoolVectorCalculator);
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
|
||||
@@ -56,10 +65,21 @@ typedef ConcatenateVectorCalculator<TfLiteTensor>
|
||||
ConcatenateTfLiteTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<Tensor> ConcatenateTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateTensorVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
|
||||
ConcatenateLandmarkVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmarkList>
|
||||
ConcatenateLandmarListVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateLandmarListVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<mediapipe::ClassificationList>
|
||||
ConcatenateClassificationListVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateClassificationListVectorCalculator);
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
|
||||
ConcatenateGlBufferVectorCalculator;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
|
||||
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
@@ -26,10 +27,10 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Concatenates several std::vector<T> following stream index order. This class
|
||||
// assumes that every input stream contains the vector<T> type. To use this
|
||||
// class for a particular type T, regisiter a calculator using
|
||||
// ConcatenateVectorCalculator<T>.
|
||||
// Concatenates several objects of type T or std::vector<T> following stream
|
||||
// index order. This class assumes that every input stream contains either T or
|
||||
// vector<T> type. To use this class for a particular type T, regisiter a
|
||||
// calculator using ConcatenateVectorCalculator<T>.
|
||||
template <typename T>
|
||||
class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
public:
|
||||
@@ -38,7 +39,8 @@ class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
RET_CHECK(cc->Outputs().NumEntries() == 1);
|
||||
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
cc->Inputs().Index(i).Set<std::vector<T>>();
|
||||
// Actual type T or vector<T> will be validated in Process().
|
||||
cc->Inputs().Index(i).SetAny();
|
||||
}
|
||||
|
||||
cc->Outputs().Index(0).Set<std::vector<T>>();
|
||||
@@ -69,9 +71,19 @@ class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
CalculatorContext* cc) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (cc->Inputs().Index(i).IsEmpty()) continue;
|
||||
const std::vector<U>& input = cc->Inputs().Index(i).Get<std::vector<U>>();
|
||||
output->insert(output->end(), input.begin(), input.end());
|
||||
auto& input = cc->Inputs().Index(i);
|
||||
|
||||
if (input.IsEmpty()) continue;
|
||||
|
||||
if (input.Value().ValidateAsType<U>().ok()) {
|
||||
const U& value = input.Get<U>();
|
||||
output->push_back(value);
|
||||
} else if (input.Value().ValidateAsType<std::vector<U>>().ok()) {
|
||||
const std::vector<U>& value = input.Get<std::vector<U>>();
|
||||
output->insert(output->end(), value.begin(), value.end());
|
||||
} else {
|
||||
return ::mediapipe::InvalidArgumentError("Invalid input stream type.");
|
||||
}
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
@@ -88,17 +100,32 @@ class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
CalculatorContext* cc) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (cc->Inputs().Index(i).IsEmpty()) continue;
|
||||
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
|
||||
cc->Inputs().Index(i).Value().Consume<std::vector<U>>();
|
||||
if (input_status.ok()) {
|
||||
std::unique_ptr<std::vector<U>> input_vector =
|
||||
std::move(input_status).ValueOrDie();
|
||||
output->insert(output->end(),
|
||||
std::make_move_iterator(input_vector->begin()),
|
||||
std::make_move_iterator(input_vector->end()));
|
||||
auto& input = cc->Inputs().Index(i);
|
||||
|
||||
if (input.IsEmpty()) continue;
|
||||
|
||||
if (input.Value().ValidateAsType<U>().ok()) {
|
||||
::mediapipe::StatusOr<std::unique_ptr<U>> value_status =
|
||||
input.Value().Consume<U>();
|
||||
if (value_status.ok()) {
|
||||
std::unique_ptr<U> value = std::move(value_status).ValueOrDie();
|
||||
output->push_back(std::move(*value));
|
||||
} else {
|
||||
return value_status.status();
|
||||
}
|
||||
} else if (input.Value().ValidateAsType<std::vector<U>>().ok()) {
|
||||
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> value_status =
|
||||
input.Value().Consume<std::vector<U>>();
|
||||
if (value_status.ok()) {
|
||||
std::unique_ptr<std::vector<U>> value =
|
||||
std::move(value_status).ValueOrDie();
|
||||
output->insert(output->end(), std::make_move_iterator(value->begin()),
|
||||
std::make_move_iterator(value->end()));
|
||||
} else {
|
||||
return value_status.status();
|
||||
}
|
||||
} else {
|
||||
return input_status.status();
|
||||
return ::mediapipe::InvalidArgumentError("Invalid input stream type.");
|
||||
}
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
@@ -109,7 +136,7 @@ class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
::mediapipe::Status ConsumeAndConcatenateVectors(std::false_type,
|
||||
CalculatorContext* cc) {
|
||||
return ::mediapipe::InternalError(
|
||||
"Cannot copy or move input vectors to concatenate them");
|
||||
"Cannot copy or move inputs to concatenate them");
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message ConcatenateVectorCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional ConcatenateVectorCalculatorOptions ext = 259397839;
|
||||
|
||||
@@ -30,11 +30,29 @@ namespace mediapipe {
|
||||
typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
|
||||
REGISTER_CALCULATOR(TestConcatenateIntVectorCalculator);
|
||||
|
||||
void AddInputVector(int index, const std::vector<int>& input, int64 timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
runner->MutableInputs()->Index(index).packets.push_back(
|
||||
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
|
||||
}
|
||||
|
||||
void AddInputVectors(const std::vector<std::vector<int>>& inputs,
|
||||
int64 timestamp, CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
runner->MutableInputs()->Index(i).packets.push_back(
|
||||
MakePacket<std::vector<int>>(inputs[i]).At(Timestamp(timestamp)));
|
||||
AddInputVector(i, inputs[i], timestamp, runner);
|
||||
}
|
||||
}
|
||||
|
||||
void AddInputItem(int index, int input, int64 timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
runner->MutableInputs()->Index(index).packets.push_back(
|
||||
MakePacket<int>(input).At(Timestamp(timestamp)));
|
||||
}
|
||||
|
||||
void AddInputItems(const std::vector<int>& inputs, int64 timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
AddInputItem(i, inputs[i], timestamp, runner);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,6 +149,135 @@ TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, ItemsOneTimestamp) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<int> inputs = {1, 2, 3};
|
||||
AddInputItems(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, ItemsTwoInputsAtTwoTimestamps) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
{
|
||||
std::vector<int> inputs = {1, 2, 3};
|
||||
AddInputItems(inputs, /*timestamp=*/1, &runner);
|
||||
}
|
||||
{
|
||||
std::vector<int> inputs = {4, 5, 6};
|
||||
AddInputItems(inputs, /*timestamp=*/2, &runner);
|
||||
}
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(2, outputs.size());
|
||||
{
|
||||
EXPECT_EQ(3, outputs[0].Get<std::vector<int>>().size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
{
|
||||
EXPECT_EQ(3, outputs[1].Get<std::vector<int>>().size());
|
||||
EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
|
||||
std::vector<int> expected_vector = {4, 5, 6};
|
||||
EXPECT_EQ(expected_vector, outputs[1].Get<std::vector<int>>());
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, ItemsOneEmptyStreamStillOutput) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
// No third input item.
|
||||
std::vector<int> inputs = {1, 2};
|
||||
AddInputItems(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, ItemsOneEmptyStreamNoOutput) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/
|
||||
"[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
|
||||
"{only_emit_if_all_present: true}",
|
||||
/*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
// No third input item.
|
||||
std::vector<int> inputs = {1, 2};
|
||||
AddInputItems(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, MixedVectorsAndItems) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/4,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<int> vector_0 = {1, 2};
|
||||
std::vector<int> vector_1 = {3, 4, 5};
|
||||
int item_0 = 6;
|
||||
int item_1 = 7;
|
||||
|
||||
AddInputVector(/*index*/ 0, vector_0, /*timestamp=*/1, &runner);
|
||||
AddInputVector(/*index*/ 1, vector_1, /*timestamp=*/1, &runner);
|
||||
AddInputItem(/*index*/ 2, item_0, /*timestamp=*/1, &runner);
|
||||
AddInputItem(/*index*/ 3, item_1, /*timestamp=*/1, &runner);
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3, 4, 5, 6, 7};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, MixedVectorsAndItemsAnother) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/4,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
int item_0 = 1;
|
||||
std::vector<int> vector_0 = {2, 3};
|
||||
std::vector<int> vector_1 = {4, 5, 6};
|
||||
int item_1 = 7;
|
||||
|
||||
AddInputItem(/*index*/ 0, item_0, /*timestamp=*/1, &runner);
|
||||
AddInputVector(/*index*/ 1, vector_0, /*timestamp=*/1, &runner);
|
||||
AddInputVector(/*index*/ 2, vector_1, /*timestamp=*/1, &runner);
|
||||
AddInputItem(/*index*/ 3, item_1, /*timestamp=*/1, &runner);
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3, 4, 5, 6, 7};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
void AddInputVectors(const std::vector<std::vector<float>>& inputs,
|
||||
int64 timestamp, CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
|
||||
@@ -17,12 +17,16 @@
|
||||
#include "mediapipe/calculators/core/constant_side_packet_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {} // namespace
|
||||
|
||||
// Generates an output side packet or multiple output side packets according to
|
||||
// the specified options.
|
||||
//
|
||||
@@ -71,6 +75,10 @@ class ConstantSidePacketCalculator : public CalculatorBase {
|
||||
packet.Set<bool>();
|
||||
} else if (packet_options.has_string_value()) {
|
||||
packet.Set<std::string>();
|
||||
} else if (packet_options.has_uint64_value()) {
|
||||
packet.Set<uint64>();
|
||||
} else if (packet_options.has_classification_list_value()) {
|
||||
packet.Set<ClassificationList>();
|
||||
} else {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"None of supported values were specified in options.");
|
||||
@@ -95,6 +103,11 @@ class ConstantSidePacketCalculator : public CalculatorBase {
|
||||
packet.Set(MakePacket<bool>(packet_options.bool_value()));
|
||||
} else if (packet_options.has_string_value()) {
|
||||
packet.Set(MakePacket<std::string>(packet_options.string_value()));
|
||||
} else if (packet_options.has_uint64_value()) {
|
||||
packet.Set(MakePacket<uint64>(packet_options.uint64_value()));
|
||||
} else if (packet_options.has_classification_list_value()) {
|
||||
packet.Set(MakePacket<ClassificationList>(
|
||||
packet_options.classification_list_value()));
|
||||
} else {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"None of supported values were specified in options.");
|
||||
|
||||
@@ -17,6 +17,9 @@ syntax = "proto2";
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
import "mediapipe/framework/formats/classification.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message ConstantSidePacketCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
@@ -29,6 +32,8 @@ message ConstantSidePacketCalculatorOptions {
|
||||
float float_value = 2;
|
||||
bool bool_value = 3;
|
||||
string string_value = 4;
|
||||
uint64 uint64_value = 5;
|
||||
ClassificationList classification_list_value = 6;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message DequantizeByteArrayCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional DequantizeByteArrayCalculatorOptions ext = 272316343;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
// Copyright 2019-2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -187,6 +187,14 @@ class GateCalculator : public CalculatorBase {
|
||||
last_gate_state_ = new_gate_state;
|
||||
|
||||
if (!allow) {
|
||||
// Close the output streams if the gate will be permanently closed.
|
||||
// Prevents buffering in calculators whose parents do no use SetOffset.
|
||||
for (int i = 0; i < num_data_streams_; ++i) {
|
||||
if (!cc->Outputs().Get("", i).IsClosed() &&
|
||||
use_side_packet_for_allow_disallow_) {
|
||||
cc->Outputs().Get("", i).Close();
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
// Copyright 2019-2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message GateCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional GateCalculatorOptions ext = 261754847;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
// Copyright 2019-2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/core/split_vector_calculator.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
@@ -233,5 +235,71 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
|
||||
kOutputName, output_fn);
|
||||
EXPECT_EQ(output, input_packets);
|
||||
}
|
||||
|
||||
constexpr char kDualInputGraphConfig[] = R"proto(
|
||||
input_stream: "input_0"
|
||||
input_stream: "input_1"
|
||||
input_stream: "input_select"
|
||||
output_stream: "test_output"
|
||||
node {
|
||||
calculator: "MuxCalculator"
|
||||
input_stream: "INPUT:0:input_0"
|
||||
input_stream: "INPUT:1:input_1"
|
||||
input_stream: "SELECT:input_select"
|
||||
output_stream: "OUTPUT:test_output"
|
||||
}
|
||||
)proto";
|
||||
|
||||
TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
|
||||
CalculatorGraphConfig config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
kDualInputGraphConfig);
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(config));
|
||||
|
||||
std::shared_ptr<int> output;
|
||||
MP_ASSERT_OK(
|
||||
graph.ObserveOutputStream("test_output", [&output](const Packet& p) {
|
||||
output = p.Get<std::shared_ptr<int>>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
auto one = std::make_shared<int>(1);
|
||||
auto two = std::make_shared<int>(2);
|
||||
auto three = std::make_shared<int>(3);
|
||||
std::weak_ptr<int> one_weak = one;
|
||||
std::weak_ptr<int> two_weak = two;
|
||||
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_0",
|
||||
MakePacket<std::shared_ptr<int>>(std::move(one)).At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_1",
|
||||
MakePacket<std::shared_ptr<int>>(std::move(two)).At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_1",
|
||||
MakePacket<std::shared_ptr<int>>(std::move(three)).At(Timestamp(1))));
|
||||
EXPECT_EQ(one, nullptr);
|
||||
EXPECT_EQ(two, nullptr);
|
||||
EXPECT_EQ(three, nullptr);
|
||||
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_select", MakePacket<int>(0).At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
EXPECT_EQ(*output, 1);
|
||||
EXPECT_NE(one_weak.lock(), nullptr);
|
||||
EXPECT_EQ(two_weak.lock(), nullptr);
|
||||
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_select", MakePacket<int>(1).At(Timestamp(1))));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
EXPECT_EQ(*output, 3);
|
||||
|
||||
MP_ASSERT_OK(graph.CloseAllInputStreams());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message PacketClonerCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional PacketClonerCalculatorOptions ext = 258872085;
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message PacketResamplerCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional PacketResamplerCalculatorOptions ext = 95743844;
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message PacketThinnerCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional PacketThinnerCalculatorOptions ext = 288533508;
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message QuantizeFloatVectorCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional QuantizeFloatVectorCalculatorOptions ext = 259848061;
|
||||
|
||||
@@ -32,6 +32,9 @@ class SequenceShiftCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
if (cc->InputSidePackets().HasTag(kPacketOffsetTag)) {
|
||||
cc->InputSidePackets().Tag(kPacketOffsetTag).Set<int>();
|
||||
}
|
||||
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
@@ -41,6 +44,8 @@ class SequenceShiftCalculator : public CalculatorBase {
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
static constexpr const char* kPacketOffsetTag = "PACKET_OFFSET";
|
||||
|
||||
// A positive offset means we want a packet to be output with the timestamp of
|
||||
// a later packet. Stores packets waiting for their output timestamps and
|
||||
// outputs a single packet when the cache fills.
|
||||
@@ -70,6 +75,9 @@ REGISTER_CALCULATOR(SequenceShiftCalculator);
|
||||
::mediapipe::Status SequenceShiftCalculator::Open(CalculatorContext* cc) {
|
||||
packet_offset_ =
|
||||
cc->Options<mediapipe::SequenceShiftCalculatorOptions>().packet_offset();
|
||||
if (cc->InputSidePackets().HasTag(kPacketOffsetTag)) {
|
||||
packet_offset_ = cc->InputSidePackets().Tag(kPacketOffsetTag).Get<int>();
|
||||
}
|
||||
cache_size_ = abs(packet_offset_);
|
||||
// An offset of zero is a no-op, but someone might still request it.
|
||||
if (packet_offset_ == 0) {
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message SequenceShiftCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional SequenceShiftCalculatorOptions ext = 107633927;
|
||||
|
||||
@@ -99,6 +99,35 @@ TEST(SequenceShiftCalculatorTest, NegativeShift) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests using a side packet to specify the offset. Shifting by -2, i.e.,
|
||||
// output input[i] with timestamp[i - 2]. The first two packets should be
|
||||
// dropped.
|
||||
TEST(SequenceShiftCalculatorTest, SidePacketOffset) {
|
||||
CalculatorGraphConfig::Node node;
|
||||
node.set_calculator("SequenceShiftCalculator");
|
||||
node.add_input_stream("input");
|
||||
node.add_output_stream("output");
|
||||
node.add_input_side_packet("PACKET_OFFSET:packet_offset");
|
||||
|
||||
CalculatorRunner runner(node);
|
||||
AddPackets(&runner);
|
||||
runner.MutableSidePackets()->Tag("PACKET_OFFSET") = Adopt(new int(-2));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& input_packets =
|
||||
runner.MutableInputs()->Index(0).packets;
|
||||
const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
|
||||
ASSERT_EQ(10, input_packets.size());
|
||||
// Input packet[i] should be output with the timestamp of input packet[i - 2].
|
||||
// The first two packets are dropped. This means timestamps match between
|
||||
// input and output packets, but the data in the output packets come from
|
||||
// input_packets[i + 2].
|
||||
ASSERT_EQ(8, output_packets.size());
|
||||
for (int i = 0; i < output_packets.size(); ++i) {
|
||||
EXPECT_EQ(input_packets[i].Timestamp(), output_packets[i].Timestamp());
|
||||
EXPECT_EQ(input_packets[i + 2].Get<int>(), output_packets[i].Get<int>());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -16,10 +16,12 @@
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
@@ -46,15 +48,18 @@ typedef SplitVectorCalculator<TfLiteTensor, false>
|
||||
SplitTfLiteTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark, false>
|
||||
typedef SplitVectorCalculator<Tensor, true> SplitTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitTensorVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<mediapipe::NormalizedLandmark, false>
|
||||
SplitLandmarkVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::NormalizedLandmarkList, false>
|
||||
typedef SplitVectorCalculator<mediapipe::NormalizedLandmarkList, false>
|
||||
SplitNormalizedLandmarkListVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitNormalizedLandmarkListVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::NormalizedRect, false>
|
||||
typedef SplitVectorCalculator<mediapipe::NormalizedRect, false>
|
||||
SplitNormalizedRectVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
|
||||
|
||||
@@ -67,8 +72,12 @@ typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true>
|
||||
REGISTER_CALCULATOR(MovableSplitGlBufferVectorCalculator);
|
||||
#endif
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::Detection, false>
|
||||
typedef SplitVectorCalculator<mediapipe::Detection, false>
|
||||
SplitDetectionVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitDetectionVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<mediapipe::ClassificationList, false>
|
||||
SplitClassificationListVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitClassificationListVectorCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -18,6 +18,8 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
// A Range {begin, end} specifies beginning ane ending indices to splice a
|
||||
// vector. A vector v is spliced to have elements v[begin:(end-1)], i.e., with
|
||||
// begin index inclusive and end index exclusive.
|
||||
|
||||
@@ -12,148 +12,78 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
exports_files(["LICENSE"])
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "opencv_image_encoder_calculator_proto",
|
||||
srcs = ["opencv_image_encoder_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "scale_image_calculator_proto",
|
||||
srcs = ["scale_image_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/framework/formats:image_format_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "set_alpha_calculator_proto",
|
||||
srcs = ["set_alpha_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "image_cropping_calculator_proto",
|
||||
srcs = ["image_cropping_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "bilateral_filter_calculator_proto",
|
||||
srcs = ["bilateral_filter_calculator.proto"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "recolor_calculator_proto",
|
||||
srcs = ["recolor_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/util:color_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "opencv_image_encoder_calculator_cc_proto",
|
||||
srcs = ["opencv_image_encoder_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [":opencv_image_encoder_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "opencv_encoded_image_to_image_frame_calculator_cc_proto",
|
||||
srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":opencv_encoded_image_to_image_frame_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "mask_overlay_calculator_cc_proto",
|
||||
srcs = ["mask_overlay_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":mask_overlay_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "scale_image_calculator_cc_proto",
|
||||
srcs = ["scale_image_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":scale_image_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "set_alpha_calculator_cc_proto",
|
||||
srcs = ["set_alpha_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":set_alpha_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "image_cropping_calculator_cc_proto",
|
||||
srcs = ["image_cropping_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":image_cropping_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "bilateral_filter_calculator_cc_proto",
|
||||
srcs = ["bilateral_filter_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [":bilateral_filter_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "recolor_calculator_cc_proto",
|
||||
srcs = ["recolor_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/util:color_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":recolor_calculator_proto"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "color_convert_calculator",
|
||||
srcs = ["color_convert_calculator.cc"],
|
||||
@@ -279,27 +209,17 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "image_transformation_calculator_proto",
|
||||
srcs = ["image_transformation_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/gpu:scale_mode_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "image_transformation_calculator_cc_proto",
|
||||
srcs = ["image_transformation_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/gpu:scale_mode_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":image_transformation_calculator_proto"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_transformation_calculator",
|
||||
srcs = ["image_transformation_calculator.cc"],
|
||||
@@ -560,32 +480,33 @@ cc_test(
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "mask_overlay_calculator_proto",
|
||||
srcs = ["mask_overlay_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "opencv_encoded_image_to_image_frame_calculator_proto",
|
||||
srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "feature_detector_calculator_proto",
|
||||
srcs = ["feature_detector_calculator.proto"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "feature_detector_calculator_cc_proto",
|
||||
srcs = ["feature_detector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":feature_detector_calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
|
||||
@@ -107,7 +107,7 @@ class BilateralFilterCalculator : public CalculatorBase {
|
||||
GLuint program_ = 0;
|
||||
GLuint vao_;
|
||||
GLuint vbo_[2]; // vertex storage
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
};
|
||||
REGISTER_CALCULATOR(BilateralFilterCalculator);
|
||||
|
||||
|
||||
@@ -519,7 +519,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
renderer = yuv_renderer_.get();
|
||||
src1 = gpu_helper_.CreateSourceTexture(input, 0);
|
||||
} else // NOLINT(readability/braces)
|
||||
#endif // iOS
|
||||
#endif // iOS
|
||||
{
|
||||
src1 = gpu_helper_.CreateSourceTexture(input);
|
||||
#if defined(TEXTURE_EXTERNAL_OES)
|
||||
@@ -531,7 +531,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
}
|
||||
renderer = ext_rgb_renderer_.get();
|
||||
} else // NOLINT(readability/braces)
|
||||
#endif // TEXTURE_EXTERNAL_OES
|
||||
#endif // TEXTURE_EXTERNAL_OES
|
||||
{
|
||||
if (!rgb_renderer_) {
|
||||
rgb_renderer_ = absl::make_unique<QuadRenderer>();
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
filegroup(
|
||||
name = "test_images",
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
licenses(["notice"])
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
load("@bazel_skylib//lib:selects.bzl", "selects")
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
selects.config_setting_group(
|
||||
name = "compute_shader_unavailable",
|
||||
match_any = [
|
||||
"//mediapipe/gpu:disable_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "inference_calculator_proto",
|
||||
srcs = ["inference_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "inference_calculator",
|
||||
srcs = ["inference_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
features = ["-layering_check"], # allow depending on inference_calculator_gpu_deps
|
||||
linkopts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-framework CoreVideo",
|
||||
"-framework MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":inference_calculator_cc_proto",
|
||||
"@com_google_absl//absl/memory",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/util:resource_util",
|
||||
"//mediapipe/util/tflite:config",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
|
||||
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
|
||||
"//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
] + select({
|
||||
":compute_shader_unavailable": [],
|
||||
"//conditions:default": [":inference_calculator_gpu_deps"],
|
||||
}) + select({
|
||||
"//conditions:default": [],
|
||||
"//mediapipe:android": [
|
||||
"//mediapipe/util/android/file/base",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/nnapi:nnapi_delegate",
|
||||
],
|
||||
}) + select({
|
||||
"//conditions:default": [
|
||||
"//mediapipe/util:cpu_util",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "inference_calculator_gpu_deps",
|
||||
deps = selects.with_or({
|
||||
"//mediapipe:ios": [
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
"//mediapipe/gpu:MPPMetalUtil",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/objc:mediapipe_framework_ios",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/metal:buffer_convert",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
|
||||
],
|
||||
"//mediapipe:macos": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/util/tflite:tflite_gpu_runner",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "tensor_converter_calculator_proto",
|
||||
srcs = ["tensor_converter_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensor_converter_calculator",
|
||||
srcs = ["tensor_converter_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
features = ["-layering_check"], # allow depending on tensor_converter_calculator_gpu_deps
|
||||
linkopts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-framework CoreVideo",
|
||||
"-framework MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":tensor_converter_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/util:resource_util",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensor_converter_calculator_gpu_deps",
|
||||
deps = select({
|
||||
"//mediapipe:android": [
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
"//mediapipe:ios": [
|
||||
"//mediapipe/gpu:MPPMetalUtil",
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
"//mediapipe/objc:mediapipe_framework_ios",
|
||||
],
|
||||
"//mediapipe:macos": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
"//mediapipe/gpu:shader_util",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensor_converter_calculator_test",
|
||||
srcs = ["tensor_converter_calculator_test.cc"],
|
||||
deps = [
|
||||
":tensor_converter_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "tensors_to_detections_calculator_proto",
|
||||
srcs = ["tensors_to_detections_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_detections_calculator",
|
||||
srcs = ["tensors_to_detections_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
features = ["-layering_check"], # allow depending on tensors_to_detections_calculator_gpu_deps
|
||||
linkopts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-framework CoreVideo",
|
||||
"-framework MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":tensors_to_detections_calculator_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
"@com_google_absl//absl/types:span",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats:location",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
] + select({
|
||||
":compute_shader_unavailable": [],
|
||||
"//conditions:default": [":tensors_to_detections_calculator_gpu_deps"],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_detections_calculator_gpu_deps",
|
||||
deps = select({
|
||||
"//mediapipe:ios": [
|
||||
"//mediapipe/gpu:MPPMetalUtil",
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
],
|
||||
"//mediapipe:macos": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "tensors_to_landmarks_calculator_proto",
|
||||
srcs = ["tensors_to_landmarks_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_landmarks_calculator",
|
||||
srcs = ["tensors_to_landmarks_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":tensors_to_landmarks_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_floats_calculator",
|
||||
srcs = ["tensors_to_floats_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_classification_calculator",
|
||||
srcs = ["tensors_to_classification_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":tensors_to_classification_calculator_cc_proto",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
"@com_google_absl//absl/types:span",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:location",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/util:resource_util",
|
||||
] + select({
|
||||
"//mediapipe:android": [
|
||||
"//mediapipe/util/android/file/base",
|
||||
],
|
||||
"//mediapipe:ios": [
|
||||
"//mediapipe/util/android/file/base",
|
||||
],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "tensors_to_classification_calculator_proto",
|
||||
srcs = ["tensors_to_classification_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensors_to_classification_calculator_test",
|
||||
srcs = ["tensors_to_classification_calculator_test.cc"],
|
||||
data = ["testdata/labelmap.txt"],
|
||||
deps = [
|
||||
":tensors_to_classification_calculator",
|
||||
":tensors_to_classification_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_googletest//:gtest_main",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_calculator",
|
||||
srcs = ["image_to_tensor_calculator.cc"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
features = ["-layering_check"], # allow depending on image_to_tensor_calculator_gpu_deps
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":image_to_tensor_calculator_cc_proto",
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_converter_opencv",
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:port",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_calculator_gpu_deps",
|
||||
deps = select({
|
||||
"//mediapipe:android": [
|
||||
":image_to_tensor_converter_gl_buffer",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
"//mediapipe:apple": [
|
||||
":image_to_tensor_converter_metal",
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
"//conditions:default": [
|
||||
":image_to_tensor_converter_gl_buffer",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "image_to_tensor_calculator_proto",
|
||||
srcs = ["image_to_tensor_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "image_to_tensor_calculator_test",
|
||||
srcs = ["image_to_tensor_calculator_test.cc"],
|
||||
data = [
|
||||
"testdata/image_to_tensor/input.jpg",
|
||||
"testdata/image_to_tensor/large_sub_rect.png",
|
||||
"testdata/image_to_tensor/large_sub_rect_keep_aspect.png",
|
||||
"testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation.png",
|
||||
"testdata/image_to_tensor/medium_sub_rect_keep_aspect.png",
|
||||
"testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation.png",
|
||||
"testdata/image_to_tensor/medium_sub_rect_with_rotation.png",
|
||||
"testdata/image_to_tensor/noop_except_range.png",
|
||||
],
|
||||
deps = [
|
||||
":image_to_tensor_calculator",
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter",
|
||||
hdrs = ["image_to_tensor_converter.h"],
|
||||
deps = [
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter_opencv",
|
||||
srcs = ["image_to_tensor_converter_opencv.cc"],
|
||||
hdrs = ["image_to_tensor_converter_opencv.h"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
deps = [
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter_gl_buffer",
|
||||
srcs = ["image_to_tensor_converter_gl_buffer.cc"],
|
||||
hdrs = ["image_to_tensor_converter_gl_buffer.h"],
|
||||
deps = ["//mediapipe/framework:port"] + select({
|
||||
"//mediapipe:apple": [],
|
||||
"//conditions:default": [
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_utils",
|
||||
"@com_google_absl//absl/strings",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_buffer_format",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:types",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:command_queue",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_call",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_texture",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:request_gpu_info",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:variable",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl/converters:util",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter_gl_texture",
|
||||
srcs = ["image_to_tensor_converter_gl_texture.cc"],
|
||||
hdrs = ["image_to_tensor_converter_gl_texture.h"],
|
||||
deps = ["//mediapipe/framework:port"] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": [
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_utils",
|
||||
"@com_google_absl//absl/strings",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:shader_util",
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_converter_metal",
|
||||
srcs = ["image_to_tensor_converter_metal.cc"],
|
||||
hdrs = ["image_to_tensor_converter_metal.h"],
|
||||
copts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
linkopts = select({
|
||||
"//mediapipe:apple": [
|
||||
"-framework CoreVideo",
|
||||
"-framework MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
deps = ["//mediapipe/framework:port"] + select({
|
||||
"//mediapipe:apple": [
|
||||
":image_to_tensor_converter",
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/gpu:MPPMetalHelper",
|
||||
"@com_google_absl//absl/strings",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_buffer_format",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:types",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_to_tensor_utils",
|
||||
srcs = ["image_to_tensor_utils.cc"],
|
||||
hdrs = ["image_to_tensor_utils.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"@com_google_absl//absl/types:optional",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "image_to_tensor_utils_test",
|
||||
srcs = ["image_to_tensor_utils_test.cc"],
|
||||
deps = [
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,275 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <array>
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_metal.h"
|
||||
#include "mediapipe/gpu/MPPMetalHelper.h"
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h"
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#else
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h"
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
namespace {
|
||||
constexpr char kInputCpu[] = "IMAGE";
|
||||
constexpr char kInputGpu[] = "IMAGE_GPU";
|
||||
constexpr char kOutputMatrix[] = "MATRIX";
|
||||
constexpr char kOutput[] = "TENSORS";
|
||||
constexpr char kInputNormRect[] = "NORM_RECT";
|
||||
constexpr char kOutputLetterboxPadding[] = "LETTERBOX_PADDING";
|
||||
} // namespace
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Converts image into Tensor, possibly with cropping, resizing and
|
||||
// normalization, according to specified inputs and options.
|
||||
//
|
||||
// Inputs:
|
||||
// IMAGE - ImageFrame [ImageFormat::SRGB/SRGBA]
|
||||
// Image to extract from.
|
||||
// IMAGE_GPU - GpuBuffer [GpuBufferFormat::kBGRA32]
|
||||
// Image to extract from.
|
||||
// (Either IMAGE or IMAGE_GPU has to be specified.)
|
||||
//
|
||||
// NORM_RECT - NormalizedRect @Optional
|
||||
// Describes region of image to extract.
|
||||
// @Optional: rect covering the whole image is used if not specified.
|
||||
//
|
||||
// Outputs:
|
||||
// TENSORS - std::vector<Tensor>
|
||||
// Vector containing a single Tensor populated with an extrated RGB image.
|
||||
// MATRIX - std::array<float, 16> @Optional
|
||||
// An std::array<float, 16> representing a 4x4 row-major-order matrix which
|
||||
// can be used to map a point on the output tensor to a point on the input
|
||||
// image.
|
||||
// LETTERBOX_PADDING - std::array<float, 4> @Optional
|
||||
// An std::array<float, 4> representing the letterbox padding from the 4
|
||||
// sides ([left, top, right, bottom]) of the output image, normalized to
|
||||
// [0.f, 1.f] by the output dimensions. The padding values are non-zero only
|
||||
// when the "keep_aspect_ratio" is true.
|
||||
//
|
||||
// For instance, when the input image is 10x10 (width x height) and the
|
||||
// output dimensions specified in the calculator option are 20x40 and
|
||||
// "keep_aspect_ratio" is true, the calculator scales the input image to
|
||||
// 20x20 and places it in the middle of the output image with an equal
|
||||
// padding of 10 pixels at the top and the bottom. The resulting array is
|
||||
// therefore [0.f, 0.25f, 0.f, 0.25f] (10/40 = 0.25f).
|
||||
//
|
||||
// Example:
|
||||
// node {
|
||||
// calculator: "ImageToTensorCalculator"
|
||||
// input_stream: "IMAGE:image" # or "IMAGE_GPU:image"
|
||||
// input_stream: "NORM_RECT:roi"
|
||||
// output_stream: "TENSORS:tensors"
|
||||
// output_stream: "MATRIX:matrix"
|
||||
// options {
|
||||
// [mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
// output_tensor_width: 256
|
||||
// output_tensor_height: 256
|
||||
// keep_aspect_ratio: false
|
||||
// output_tensor_float_range {
|
||||
// min: 0.0
|
||||
// max: 1.0
|
||||
// }
|
||||
// # gpu_origin: CONVENTIONAL # or TOP_LEFT
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class ImageToTensorCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
const auto& options =
|
||||
cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
|
||||
|
||||
RET_CHECK(options.has_output_tensor_float_range())
|
||||
<< "Output tensor range is required.";
|
||||
RET_CHECK_LT(options.output_tensor_float_range().min(),
|
||||
options.output_tensor_float_range().max())
|
||||
<< "Valid output tensor range is required.";
|
||||
RET_CHECK_GT(options.output_tensor_width(), 0)
|
||||
<< "Valid output tensor width is required.";
|
||||
RET_CHECK_GT(options.output_tensor_height(), 0)
|
||||
<< "Valid output tensor height is required.";
|
||||
|
||||
if (cc->Inputs().HasTag(kInputNormRect)) {
|
||||
cc->Inputs().Tag(kInputNormRect).Set<mediapipe::NormalizedRect>();
|
||||
}
|
||||
if (cc->Outputs().HasTag(kOutputLetterboxPadding)) {
|
||||
cc->Outputs().Tag(kOutputLetterboxPadding).Set<std::array<float, 4>>();
|
||||
}
|
||||
if (cc->Outputs().HasTag(kOutputMatrix)) {
|
||||
cc->Outputs().Tag(kOutputMatrix).Set<std::array<float, 16>>();
|
||||
}
|
||||
|
||||
const bool has_cpu_input = cc->Inputs().HasTag(kInputCpu);
|
||||
const bool has_gpu_input = cc->Inputs().HasTag(kInputGpu);
|
||||
RET_CHECK_EQ((has_cpu_input ? 1 : 0) + (has_gpu_input ? 1 : 0), 1)
|
||||
<< "Either CPU or GPU input is expected, not both.";
|
||||
|
||||
if (has_cpu_input) {
|
||||
cc->Inputs().Tag(kInputCpu).Set<mediapipe::ImageFrame>();
|
||||
} else if (has_gpu_input) {
|
||||
#if MEDIAPIPE_DISABLE_GPU
|
||||
return mediapipe::UnimplementedError("GPU processing is disabled");
|
||||
#else
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
|
||||
#else
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
cc->Inputs().Tag(kInputGpu).Set<mediapipe::GpuBuffer>();
|
||||
|
||||
#endif // MEDIAPIPE_DISABLE_GPU
|
||||
}
|
||||
cc->Outputs().Tag(kOutput).Set<std::vector<Tensor>>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) {
|
||||
// Makes sure outputs' next timestamp bound update is handled automatically
|
||||
// by the framework.
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
options_ = cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
|
||||
output_width_ = options_.output_tensor_width();
|
||||
output_height_ = options_.output_tensor_height();
|
||||
range_min_ = options_.output_tensor_float_range().min();
|
||||
range_max_ = options_.output_tensor_float_range().max();
|
||||
|
||||
if (cc->Inputs().HasTag(kInputCpu)) {
|
||||
ASSIGN_OR_RETURN(converter_, CreateOpenCvConverter(cc));
|
||||
} else {
|
||||
#if MEDIAPIPE_DISABLE_GPU
|
||||
return mediapipe::UnimplementedError("GPU processing is disabled");
|
||||
#else
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
ASSIGN_OR_RETURN(converter_, CreateMetalConverter(cc));
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
ASSIGN_OR_RETURN(converter_, CreateImageToGlBufferTensorConverter(
|
||||
cc, DoesInputStartAtBottom()));
|
||||
#else
|
||||
ASSIGN_OR_RETURN(converter_, CreateImageToGlTextureTensorConverter(
|
||||
cc, DoesInputStartAtBottom()));
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
|
||||
#endif // MEDIAPIPE_DISABLE_GPU
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) {
|
||||
const InputStreamShard& input = cc->Inputs().Tag(
|
||||
cc->Inputs().HasTag(kInputCpu) ? kInputCpu : kInputGpu);
|
||||
if (input.IsEmpty()) {
|
||||
// Timestamp bound update happens automatically. (See Open().)
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
absl::optional<mediapipe::NormalizedRect> norm_rect;
|
||||
if (cc->Inputs().HasTag(kInputNormRect)) {
|
||||
if (cc->Inputs().Tag(kInputNormRect).IsEmpty()) {
|
||||
// Timestamp bound update happens automatically. (See Open().)
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
norm_rect =
|
||||
cc->Inputs().Tag(kInputNormRect).Get<mediapipe::NormalizedRect>();
|
||||
if (norm_rect->width() == 0 && norm_rect->height() == 0) {
|
||||
// WORKAROUND: some existing graphs may use sentinel rects {width=0,
|
||||
// height=0, ...} quite often and calculator has to handle them
|
||||
// gracefully by updating timestamp bound instead of returning failure.
|
||||
// Timestamp bound update happens automatically. (See Open().)
|
||||
// NOTE: usage of sentinel rects should be avoided.
|
||||
DLOG(WARNING)
|
||||
<< "Updating timestamp bound in response to a sentinel rect";
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
}
|
||||
|
||||
const Packet& image_packet = input.Value();
|
||||
const Size& size = converter_->GetImageSize(image_packet);
|
||||
RotatedRect roi = GetRoi(size.width, size.height, norm_rect);
|
||||
ASSIGN_OR_RETURN(auto padding, PadRoi(options_.output_tensor_width(),
|
||||
options_.output_tensor_height(),
|
||||
options_.keep_aspect_ratio(), &roi));
|
||||
if (cc->Outputs().HasTag(kOutputLetterboxPadding)) {
|
||||
cc->Outputs()
|
||||
.Tag(kOutputLetterboxPadding)
|
||||
.AddPacket(MakePacket<std::array<float, 4>>(padding).At(
|
||||
cc->InputTimestamp()));
|
||||
}
|
||||
if (cc->Outputs().HasTag(kOutputMatrix)) {
|
||||
std::array<float, 16> matrix;
|
||||
GetRotatedSubRectToRectTransformMatrix(roi, size.width, size.height,
|
||||
/*flip_horizontaly=*/false,
|
||||
&matrix);
|
||||
cc->Outputs()
|
||||
.Tag(kOutputMatrix)
|
||||
.AddPacket(MakePacket<std::array<float, 16>>(std::move(matrix))
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
ASSIGN_OR_RETURN(
|
||||
Tensor tensor,
|
||||
converter_->Convert(image_packet, roi, {output_width_, output_height_},
|
||||
range_min_, range_max_));
|
||||
|
||||
std::vector<Tensor> result;
|
||||
result.push_back(std::move(tensor));
|
||||
cc->Outputs().Tag(kOutput).AddPacket(
|
||||
MakePacket<std::vector<Tensor>>(std::move(result))
|
||||
.At(cc->InputTimestamp()));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
bool DoesInputStartAtBottom() {
|
||||
return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
|
||||
}
|
||||
|
||||
std::unique_ptr<ImageToTensorConverter> converter_;
|
||||
mediapipe::ImageToTensorCalculatorOptions options_;
|
||||
int output_width_ = 0;
|
||||
int output_height_ = 0;
|
||||
float range_min_ = 0.0f;
|
||||
float range_max_ = 1.0f;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(ImageToTensorCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message GpuOrigin {
|
||||
enum Mode {
|
||||
DEFAULT = 0;
|
||||
|
||||
// OpenGL: bottom-left origin
|
||||
// Metal : top-left origin
|
||||
CONVENTIONAL = 1;
|
||||
|
||||
// OpenGL: top-left origin
|
||||
// Metal : top-left origin
|
||||
TOP_LEFT = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message ImageToTensorCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional ImageToTensorCalculatorOptions ext = 334361939;
|
||||
}
|
||||
|
||||
// Range of float values [min, max].
|
||||
// min, must be strictly less than max.
|
||||
message FloatRange {
|
||||
optional float min = 1;
|
||||
optional float max = 2;
|
||||
}
|
||||
|
||||
optional int32 output_tensor_width = 1;
|
||||
optional int32 output_tensor_height = 2;
|
||||
|
||||
// If true, image region will be extracted and copied into tensor keeping
|
||||
// region aspect ratio, which usually results in letterbox padding. Otherwise,
|
||||
// if false, image region is stretched to fill output tensor fully.
|
||||
optional bool keep_aspect_ratio = 3;
|
||||
|
||||
// Output tensor element range/type image pixels are converted to.
|
||||
oneof range {
|
||||
FloatRange output_tensor_float_range = 4;
|
||||
}
|
||||
|
||||
// For CONVENTIONAL mode for OpenGL, input image starts at bottom and needs
|
||||
// to be flipped vertically as tensors are expected to start at top.
|
||||
// (DEFAULT or unset interpreted as CONVENTIONAL.)
|
||||
optional GpuOrigin.Mode gpu_origin = 5;
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
cv::Mat GetRgb(absl::string_view path) {
|
||||
cv::Mat bgr = cv::imread(file::JoinPath("./", path));
|
||||
cv::Mat rgb;
|
||||
cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGB);
|
||||
return rgb;
|
||||
}
|
||||
|
||||
cv::Mat GetRgba(absl::string_view path) {
|
||||
cv::Mat bgr = cv::imread(file::JoinPath("./", path));
|
||||
cv::Mat rgb;
|
||||
cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGBA);
|
||||
return rgb;
|
||||
}
|
||||
|
||||
// Image to tensor test template.
|
||||
// No processing/assertions should be done after the function is invoked.
|
||||
void RunTest(cv::Mat input, cv::Mat expected_result, float range_min,
|
||||
float range_max, int tensor_width, int tensor_height,
|
||||
bool keep_aspect, const mediapipe::NormalizedRect& roi) {
|
||||
auto graph_config = mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
absl::Substitute(R"(
|
||||
input_stream: "input_image"
|
||||
input_stream: "roi"
|
||||
node {
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_stream: "NORM_RECT:roi"
|
||||
output_stream: "TENSORS:tensor"
|
||||
options {
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: $0
|
||||
output_tensor_height: $1
|
||||
keep_aspect_ratio: $4
|
||||
output_tensor_float_range {
|
||||
min: $2
|
||||
max: $3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)",
|
||||
/*$0=*/tensor_width,
|
||||
/*$1=*/tensor_height,
|
||||
/*$2=*/range_min,
|
||||
/*$3=*/range_max,
|
||||
/*$4=*/keep_aspect ? "true" : "false"));
|
||||
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
ImageFrame input_image(
|
||||
input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB,
|
||||
input.cols, input.rows, input.step, input.data, [](uint8*) {});
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_image",
|
||||
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"roi",
|
||||
MakePacket<mediapipe::NormalizedRect>(std::move(roi)).At(Timestamp(0))));
|
||||
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
ASSERT_THAT(output_packets, testing::SizeIs(1));
|
||||
|
||||
// Get and process results.
|
||||
const std::vector<Tensor>& tensor_vec =
|
||||
output_packets[0].Get<std::vector<Tensor>>();
|
||||
ASSERT_THAT(tensor_vec, testing::SizeIs(1));
|
||||
|
||||
const Tensor& tensor = tensor_vec[0];
|
||||
EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kFloat32);
|
||||
|
||||
auto view = tensor.GetCpuReadView();
|
||||
cv::Mat tensor_mat(tensor_height, tensor_width, CV_32FC3,
|
||||
const_cast<float*>(view.buffer<float>()));
|
||||
cv::Mat result_rgb;
|
||||
auto transformation =
|
||||
GetValueRangeTransformation(range_min, range_max, 0.0f, 255.0f)
|
||||
.ValueOrDie();
|
||||
tensor_mat.convertTo(result_rgb, CV_8UC3, transformation.scale,
|
||||
transformation.offset);
|
||||
|
||||
cv::Mat diff;
|
||||
cv::absdiff(result_rgb, expected_result, diff);
|
||||
double max_val;
|
||||
cv::minMaxLoc(diff, nullptr, &max_val);
|
||||
// Expects the maximum absolute pixel-by-pixel difference is less than 5.
|
||||
EXPECT_LE(max_val, 5);
|
||||
|
||||
// Fully close graph at end, otherwise calculator+tensors are destroyed
|
||||
// after calling WaitUntilDone().
|
||||
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
|
||||
MP_ASSERT_OK(graph.CloseInputStream("roi"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspect) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.65f);
|
||||
roi.set_y_center(0.4f);
|
||||
roi.set_width(0.5f);
|
||||
roi.set_height(0.5f);
|
||||
roi.set_rotation(0);
|
||||
RunTest(
|
||||
GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg"),
|
||||
GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png"),
|
||||
/*range_min=*/0.0f,
|
||||
/*range_max=*/1.0f,
|
||||
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true, roi);
|
||||
}
|
||||
|
||||
TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectWithRotation) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.65f);
|
||||
roi.set_y_center(0.4f);
|
||||
roi.set_width(0.5f);
|
||||
roi.set_height(0.5f);
|
||||
roi.set_rotation(M_PI * 90.0f / 180.0f);
|
||||
RunTest(GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg"),
|
||||
GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/"
|
||||
"medium_sub_rect_keep_aspect_with_rotation.png"),
|
||||
/*range_min=*/0.0f, /*range_max=*/1.0f,
|
||||
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
|
||||
roi);
|
||||
}
|
||||
|
||||
TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotation) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.65f);
|
||||
roi.set_y_center(0.4f);
|
||||
roi.set_width(0.5f);
|
||||
roi.set_height(0.5f);
|
||||
roi.set_rotation(M_PI * -45.0f / 180.0f);
|
||||
RunTest(
|
||||
GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg"),
|
||||
GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png"),
|
||||
/*range_min=*/-1.0f,
|
||||
/*range_max=*/1.0f,
|
||||
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false, roi);
|
||||
}
|
||||
|
||||
TEST(ImageToTensorCalculatorTest, LargeSubRect) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.5f);
|
||||
roi.set_height(1.1f);
|
||||
roi.set_rotation(0);
|
||||
RunTest(GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg"),
|
||||
GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/large_sub_rect.png"),
|
||||
/*range_min=*/0.0f,
|
||||
/*range_max=*/1.0f,
|
||||
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
|
||||
roi);
|
||||
}
|
||||
|
||||
TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.5f);
|
||||
roi.set_height(1.1f);
|
||||
roi.set_rotation(0);
|
||||
RunTest(
|
||||
GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg"),
|
||||
GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png"),
|
||||
/*range_min=*/0.0f,
|
||||
/*range_max=*/1.0f,
|
||||
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true, roi);
|
||||
}
|
||||
|
||||
TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotation) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.5f);
|
||||
roi.set_height(1.1f);
|
||||
roi.set_rotation(M_PI * -15.0f / 180.0f);
|
||||
RunTest(GetRgba("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg"),
|
||||
GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/"
|
||||
"large_sub_rect_keep_aspect_with_rotation.png"),
|
||||
/*range_min=*/0.0f,
|
||||
/*range_max=*/1.0f,
|
||||
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
|
||||
roi);
|
||||
}
|
||||
|
||||
TEST(ImageToTensorCalculatorTest, NoOpExceptRange) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.0f);
|
||||
roi.set_height(1.0f);
|
||||
roi.set_rotation(0);
|
||||
RunTest(GetRgba("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg"),
|
||||
GetRgb("/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/noop_except_range.png"),
|
||||
/*range_min=*/0.0f,
|
||||
/*range_max=*/1.0f,
|
||||
/*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
|
||||
roi);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,53 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_H_
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
struct Size {
|
||||
int width;
|
||||
int height;
|
||||
};
|
||||
|
||||
// Converts image to tensor.
|
||||
class ImageToTensorConverter {
|
||||
public:
|
||||
virtual ~ImageToTensorConverter() = default;
|
||||
|
||||
virtual Size GetImageSize(const Packet& image_packet) = 0;
|
||||
|
||||
// Converts image to tensor.
|
||||
// @image_packet contains image to extract from.
|
||||
// @roi describes region of interest within the image to extract (absolute
|
||||
// values).
|
||||
// @output_dims dimensions of output tensor.
|
||||
// @range_min/max describes output tensor range image pixels should converted
|
||||
// to.
|
||||
virtual ::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
|
||||
const RotatedRect& roi,
|
||||
const Size& output_dims,
|
||||
float range_min,
|
||||
float range_max) = 0;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_H_
|
||||
@@ -0,0 +1,340 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h"
|
||||
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
|
||||
#include <array>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "tensorflow/lite/delegates/gpu/common/shape.h"
|
||||
#include "tensorflow/lite/delegates/gpu/common/types.h"
|
||||
#include "tensorflow/lite/delegates/gpu/gl/command_queue.h"
|
||||
#include "tensorflow/lite/delegates/gpu/gl/converters/util.h"
|
||||
#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
|
||||
#include "tensorflow/lite/delegates/gpu/gl/gl_call.h"
|
||||
#include "tensorflow/lite/delegates/gpu/gl/gl_texture.h"
|
||||
#include "tensorflow/lite/delegates/gpu/gl/request_gpu_info.h"
|
||||
#include "tensorflow/lite/delegates/gpu/gl/variable.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
// Implements a common pattern of extracting a subrect from RGBA input texture
|
||||
// and resizing it into a buffer.
|
||||
class SubRectExtractorGl {
|
||||
public:
|
||||
// Extracts a region defined by @sub_rect, removes A channel, transforms input
|
||||
// pixels as alpha * x + beta and resizes result into destination.
|
||||
::mediapipe::Status ExtractSubRectToBuffer(
|
||||
const tflite::gpu::gl::GlTexture& texture,
|
||||
const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect,
|
||||
bool flip_horizontaly, float alpha, float beta,
|
||||
const tflite::gpu::HW& destination_size,
|
||||
tflite::gpu::gl::CommandQueue* command_queue,
|
||||
tflite::gpu::gl::GlBuffer* destination);
|
||||
|
||||
static ::mediapipe::StatusOr<SubRectExtractorGl> Create(
|
||||
bool input_starts_at_bottom);
|
||||
|
||||
private:
|
||||
explicit SubRectExtractorGl(tflite::gpu::gl::GlProgram program,
|
||||
tflite::gpu::uint3 workgroup_size)
|
||||
: program_(std::move(program)), workgroup_size_(workgroup_size) {}
|
||||
|
||||
tflite::gpu::gl::GlProgram program_;
|
||||
tflite::gpu::uint3 workgroup_size_;
|
||||
};
|
||||
|
||||
::mediapipe::Status SetMat4x4(const tflite::gpu::gl::GlProgram& program,
|
||||
const std::string& name, float* data) {
|
||||
GLint uniform_id;
|
||||
MP_RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glGetUniformLocation, &uniform_id,
|
||||
program.id(), name.c_str()));
|
||||
return TFLITE_GPU_CALL_GL(glProgramUniformMatrix4fv, program.id(), uniform_id,
|
||||
1, GL_TRUE, data);
|
||||
}
|
||||
|
||||
class GlParametersOverride {
|
||||
public:
|
||||
static ::mediapipe::StatusOr<GlParametersOverride> Create(
|
||||
const std::vector<std::pair<GLenum, GLint>>& overrides) {
|
||||
std::vector<GLint> old_values(overrides.size());
|
||||
for (int i = 0; i < overrides.size(); ++i) {
|
||||
MP_RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glGetTexParameteriv, GL_TEXTURE_2D,
|
||||
overrides[i].first,
|
||||
&old_values[i]));
|
||||
if (overrides[i].second != old_values[i]) {
|
||||
MP_RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glTexParameteri, GL_TEXTURE_2D,
|
||||
overrides[i].first,
|
||||
overrides[i].second));
|
||||
}
|
||||
}
|
||||
return GlParametersOverride(overrides, std::move(old_values));
|
||||
}
|
||||
|
||||
::mediapipe::Status Revert() {
|
||||
for (int i = 0; i < overrides_.size(); ++i) {
|
||||
if (overrides_[i].second != old_values_[i]) {
|
||||
MP_RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glTexParameteri, GL_TEXTURE_2D,
|
||||
overrides_[i].first,
|
||||
old_values_[i]));
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
GlParametersOverride(const std::vector<std::pair<GLenum, GLint>>& overrides,
|
||||
std::vector<GLint> old_values)
|
||||
: overrides_(overrides), old_values_(std::move(old_values)) {}
|
||||
|
||||
std::vector<std::pair<GLenum, GLint>> overrides_;
|
||||
std::vector<GLint> old_values_;
|
||||
};
|
||||
|
||||
constexpr char kShaderCode[] = R"(
|
||||
layout(std430) buffer;
|
||||
|
||||
precision highp float;
|
||||
|
||||
// It is possible to use "vec3 elements[];" here, however due to alignment
|
||||
// requirements it works only when "packed" layout is used. "packed" layout is
|
||||
// determined by implementation and it's expected that OpenGL API is used to
|
||||
// query the layout. Favoring float array over vec3, considering performance is
|
||||
// comparable, layout is the same and no need for layout querying (even though
|
||||
// it's not quite needed here as there's only one member).
|
||||
layout(binding = 0) writeonly buffer B0 {
|
||||
float elements[];
|
||||
} output_data;
|
||||
|
||||
uniform ivec2 out_size;
|
||||
uniform float alpha;
|
||||
uniform float beta;
|
||||
uniform mat4 transform_matrix;
|
||||
uniform mediump sampler2D input_data;
|
||||
|
||||
void main() {
|
||||
int out_width = out_size.x;
|
||||
int out_height = out_size.y;
|
||||
|
||||
ivec2 gid = ivec2(gl_GlobalInvocationID.xy);
|
||||
if (gid.x >= out_width || gid.y >= out_height) {
|
||||
return;
|
||||
}
|
||||
|
||||
// transform from image.width, image.height range to [0, 1]
|
||||
float normal_x = (float(gid.x) + 0.5f) / float(out_width);
|
||||
float normal_y = (float(gid.y) + 0.5f) / float(out_height);
|
||||
vec4 tc = vec4(normal_x, normal_y, 0.0, 1.0);
|
||||
|
||||
// Apply transformation from roi coordinates to original image coordinates.
|
||||
tc = transform_matrix * tc;
|
||||
#ifdef INPUT_STARTS_AT_BOTTOM
|
||||
// Opengl texture sampler has origin in lower left corner,
|
||||
// so we invert y coordinate.
|
||||
tc.y = 1.0f - tc.y;
|
||||
#endif // INPUT_STARTS_AT_BOTTOM
|
||||
vec4 src_value = alpha * texture(input_data, tc.xy) + beta;
|
||||
|
||||
int linear_index = gid.y * out_width + gid.x;
|
||||
|
||||
// output_data.elements is populated as though it contains vec3 elements.
|
||||
int first_component_index = 3 * linear_index;
|
||||
output_data.elements[first_component_index] = src_value.r;
|
||||
output_data.elements[first_component_index + 1] = src_value.g;
|
||||
output_data.elements[first_component_index + 2] = src_value.b;
|
||||
}
|
||||
)";
|
||||
|
||||
::mediapipe::Status SubRectExtractorGl::ExtractSubRectToBuffer(
|
||||
const tflite::gpu::gl::GlTexture& texture,
|
||||
const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect,
|
||||
bool flip_horizontaly, float alpha, float beta,
|
||||
const tflite::gpu::HW& destination_size,
|
||||
tflite::gpu::gl::CommandQueue* command_queue,
|
||||
tflite::gpu::gl::GlBuffer* destination) {
|
||||
std::array<float, 16> transform_mat;
|
||||
GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w,
|
||||
texture_size.h, flip_horizontaly,
|
||||
&transform_mat);
|
||||
MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0));
|
||||
|
||||
ASSIGN_OR_RETURN(auto overrides, GlParametersOverride::Create(
|
||||
{{GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE},
|
||||
{GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE},
|
||||
{GL_TEXTURE_MIN_FILTER, GL_LINEAR},
|
||||
{GL_TEXTURE_MAG_FILTER, GL_LINEAR}}));
|
||||
|
||||
MP_RETURN_IF_ERROR(destination->BindToIndex(0));
|
||||
MP_RETURN_IF_ERROR(program_.SetParameter({"input_data", 0}));
|
||||
MP_RETURN_IF_ERROR(
|
||||
SetMat4x4(program_, "transform_matrix", transform_mat.data()));
|
||||
MP_RETURN_IF_ERROR(program_.SetParameter(
|
||||
{"out_size", tflite::gpu::int2(destination_size.w, destination_size.h)}));
|
||||
MP_RETURN_IF_ERROR(program_.SetParameter({"alpha", alpha}));
|
||||
MP_RETURN_IF_ERROR(program_.SetParameter({"beta", beta}));
|
||||
tflite::gpu::uint3 num_workgroups = tflite::gpu::DivideRoundUp(
|
||||
tflite::gpu::uint3{destination_size.w, destination_size.h, 1},
|
||||
workgroup_size_);
|
||||
MP_RETURN_IF_ERROR(command_queue->Dispatch(program_, num_workgroups));
|
||||
|
||||
return overrides.Revert();
|
||||
}
|
||||
|
||||
::mediapipe::StatusOr<SubRectExtractorGl> SubRectExtractorGl::Create(
|
||||
bool input_starts_at_bottom) {
|
||||
const tflite::gpu::uint3 workgroup_size = {8, 8, 1};
|
||||
std::string starts_at_bottom_def;
|
||||
if (input_starts_at_bottom) {
|
||||
starts_at_bottom_def = R"(
|
||||
#define INPUT_STARTS_AT_BOTTOM;
|
||||
)";
|
||||
}
|
||||
const std::string full_shader_source =
|
||||
absl::StrCat(tflite::gpu::gl::GetShaderHeader(workgroup_size),
|
||||
starts_at_bottom_def, kShaderCode);
|
||||
|
||||
tflite::gpu::gl::GlShader shader;
|
||||
MP_RETURN_IF_ERROR(tflite::gpu::gl::GlShader::CompileShader(
|
||||
GL_COMPUTE_SHADER, full_shader_source, &shader));
|
||||
tflite::gpu::gl::GlProgram program;
|
||||
MP_RETURN_IF_ERROR(
|
||||
tflite::gpu::gl::GlProgram::CreateWithShader(shader, &program));
|
||||
|
||||
return SubRectExtractorGl(std::move(program), workgroup_size);
|
||||
}
|
||||
|
||||
class GlProcessor : public ImageToTensorConverter {
|
||||
public:
|
||||
::mediapipe::Status Init(CalculatorContext* cc, bool input_starts_at_bottom) {
|
||||
MP_RETURN_IF_ERROR(gl_helper_.Open(cc));
|
||||
return gl_helper_.RunInGlContext(
|
||||
[this, input_starts_at_bottom]() -> ::mediapipe::Status {
|
||||
tflite::gpu::GpuInfo gpu_info;
|
||||
MP_RETURN_IF_ERROR(tflite::gpu::gl::RequestGpuInfo(&gpu_info));
|
||||
RET_CHECK(tflite::gpu::IsOpenGl31OrAbove(gpu_info))
|
||||
<< "OpenGL ES 3.1 is required.";
|
||||
command_queue_ = tflite::gpu::gl::NewCommandQueue(gpu_info);
|
||||
|
||||
ASSIGN_OR_RETURN(auto extractor,
|
||||
SubRectExtractorGl::Create(input_starts_at_bottom));
|
||||
extractor_ =
|
||||
absl::make_unique<SubRectExtractorGl>(std::move(extractor));
|
||||
return ::mediapipe::OkStatus();
|
||||
});
|
||||
}
|
||||
|
||||
Size GetImageSize(const Packet& image_packet) override {
|
||||
const auto& image = image_packet.Get<mediapipe::GpuBuffer>();
|
||||
return {image.width(), image.height()};
|
||||
}
|
||||
|
||||
::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
|
||||
const RotatedRect& roi,
|
||||
const Size& output_dims,
|
||||
float range_min,
|
||||
float range_max) override {
|
||||
const auto& input = image_packet.Get<mediapipe::GpuBuffer>();
|
||||
if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) {
|
||||
return InvalidArgumentError(
|
||||
absl::StrCat("Only BGRA/RGBA textures are supported, passed format: ",
|
||||
static_cast<uint32_t>(input.format())));
|
||||
}
|
||||
|
||||
constexpr int kNumChannels = 3;
|
||||
Tensor tensor(Tensor::ElementType::kFloat32,
|
||||
{1, output_dims.height, output_dims.width, kNumChannels});
|
||||
|
||||
MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext(
|
||||
[this, &tensor, &input, &roi, &output_dims, range_min,
|
||||
range_max]() -> ::mediapipe::Status {
|
||||
constexpr int kRgbaNumChannels = 4;
|
||||
auto source_texture = gl_helper_.CreateSourceTexture(input);
|
||||
tflite::gpu::gl::GlTexture input_texture(
|
||||
GL_TEXTURE_2D, source_texture.name(), GL_RGBA,
|
||||
source_texture.width() * source_texture.height() *
|
||||
kRgbaNumChannels * sizeof(uint8_t),
|
||||
/*layer=*/0,
|
||||
/*owned=*/false);
|
||||
|
||||
constexpr float kInputImageRangeMin = 0.0f;
|
||||
constexpr float kInputImageRangeMax = 1.0f;
|
||||
ASSIGN_OR_RETURN(auto transform,
|
||||
GetValueRangeTransformation(kInputImageRangeMin,
|
||||
kInputImageRangeMax,
|
||||
range_min, range_max));
|
||||
|
||||
auto buffer_view = tensor.GetOpenGlBufferWriteView();
|
||||
tflite::gpu::gl::GlBuffer output(GL_SHADER_STORAGE_BUFFER,
|
||||
buffer_view.name(), tensor.bytes(),
|
||||
/*offset=*/0,
|
||||
/*has_ownership=*/false);
|
||||
MP_RETURN_IF_ERROR(extractor_->ExtractSubRectToBuffer(
|
||||
input_texture,
|
||||
tflite::gpu::HW(source_texture.height(), source_texture.width()),
|
||||
roi,
|
||||
/*flip_horizontaly=*/false, transform.scale, transform.offset,
|
||||
tflite::gpu::HW(output_dims.height, output_dims.width),
|
||||
command_queue_.get(), &output));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
|
||||
return tensor;
|
||||
}
|
||||
|
||||
~GlProcessor() override {
|
||||
gl_helper_.RunInGlContext([this]() {
|
||||
// Release OpenGL resources.
|
||||
extractor_ = nullptr;
|
||||
command_queue_ = nullptr;
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<tflite::gpu::gl::CommandQueue> command_queue_;
|
||||
std::unique_ptr<SubRectExtractorGl> extractor_;
|
||||
mediapipe::GlCalculatorHelper gl_helper_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
|
||||
CreateImageToGlBufferTensorConverter(CalculatorContext* cc,
|
||||
bool input_starts_at_bottom) {
|
||||
auto result = absl::make_unique<GlProcessor>();
|
||||
MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom));
|
||||
|
||||
// Simply "return std::move(result)" failed to build on macOS with bazel.
|
||||
return std::unique_ptr<ImageToTensorConverter>(std::move(result));
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_BUFFER_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_BUFFER_H_
|
||||
|
||||
#include "mediapipe/framework/port.h"
|
||||
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Creates image to tensor (represented as OpenGL buffer) converter.
|
||||
// NOTE: mediapipe::GlCalculatorHelper::UpdateContract invocation must precede
|
||||
// converter creation.
|
||||
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
|
||||
CreateImageToGlBufferTensorConverter(CalculatorContext* cc,
|
||||
bool input_starts_at_bottom);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_BUFFER_H_
|
||||
@@ -0,0 +1,323 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h"
|
||||
|
||||
#include "mediapipe/framework/port.h"
|
||||
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
|
||||
|
||||
#include <array>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#include "mediapipe/gpu/gl_simple_shaders.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "mediapipe/gpu/shader_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
class GlParametersOverride {
|
||||
public:
|
||||
static ::mediapipe::StatusOr<GlParametersOverride> Create(
|
||||
const std::vector<std::pair<GLenum, GLint>>& overrides) {
|
||||
std::vector<GLint> old_values(overrides.size());
|
||||
for (int i = 0; i < overrides.size(); ++i) {
|
||||
glGetTexParameteriv(GL_TEXTURE_2D, overrides[i].first, &old_values[i]);
|
||||
if (overrides[i].second != old_values[i]) {
|
||||
glTexParameteri(GL_TEXTURE_2D, overrides[i].first, overrides[i].second);
|
||||
}
|
||||
}
|
||||
return GlParametersOverride(overrides, std::move(old_values));
|
||||
}
|
||||
|
||||
::mediapipe::Status Revert() {
|
||||
for (int i = 0; i < overrides_.size(); ++i) {
|
||||
if (overrides_[i].second != old_values_[i]) {
|
||||
glTexParameteri(GL_TEXTURE_2D, overrides_[i].first, old_values_[i]);
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
GlParametersOverride(const std::vector<std::pair<GLenum, GLint>>& overrides,
|
||||
std::vector<GLint> old_values)
|
||||
: overrides_(overrides), old_values_(std::move(old_values)) {}
|
||||
|
||||
std::vector<std::pair<GLenum, GLint>> overrides_;
|
||||
std::vector<GLint> old_values_;
|
||||
};
|
||||
|
||||
constexpr int kAttribVertex = 0;
|
||||
constexpr int kAttribTexturePosition = 1;
|
||||
constexpr int kNumAttributes = 2;
|
||||
|
||||
class GlProcessor : public ImageToTensorConverter {
|
||||
public:
|
||||
::mediapipe::Status Init(CalculatorContext* cc, bool input_starts_at_bottom) {
|
||||
MP_RETURN_IF_ERROR(gl_helper_.Open(cc));
|
||||
return gl_helper_.RunInGlContext([this, input_starts_at_bottom]()
|
||||
-> ::mediapipe::Status {
|
||||
const GLint attr_location[kNumAttributes] = {
|
||||
kAttribVertex,
|
||||
kAttribTexturePosition,
|
||||
};
|
||||
const GLchar* attr_name[kNumAttributes] = {
|
||||
"position",
|
||||
"texture_coordinate",
|
||||
};
|
||||
|
||||
constexpr GLchar kExtractSubRectVertexShader[] = R"(
|
||||
in vec4 position;
|
||||
in mediump vec4 texture_coordinate;
|
||||
out mediump vec2 sample_coordinate;
|
||||
uniform mat4 transform_matrix;
|
||||
|
||||
void main() {
|
||||
gl_Position = position;
|
||||
// Apply transformation from roi coordinates to original image coordinates.
|
||||
vec4 tc = transform_matrix * texture_coordinate;
|
||||
#ifdef INPUT_STARTS_AT_BOTTOM
|
||||
// Opengl texture sampler has origin in lower left corner,
|
||||
// so we invert y coordinate.
|
||||
tc.y = 1.0 - tc.y;
|
||||
#endif // defined(INPUT_STARTS_AT_BOTTOM)
|
||||
sample_coordinate = tc.xy;
|
||||
}
|
||||
)";
|
||||
|
||||
constexpr GLchar kExtractSubRectFragBody[] = R"(
|
||||
DEFAULT_PRECISION(mediump, float)
|
||||
|
||||
// Provided by kExtractSubRectVertexShader.
|
||||
in vec2 sample_coordinate;
|
||||
|
||||
uniform sampler2D input_texture;
|
||||
uniform float alpha;
|
||||
uniform float beta;
|
||||
|
||||
#ifdef GL_ES
|
||||
#define fragColor gl_FragColor
|
||||
#else
|
||||
out vec4 fragColor;
|
||||
#endif // defined(GL_ES);
|
||||
|
||||
void main() {
|
||||
fragColor = alpha * texture2D(input_texture, sample_coordinate) + beta;
|
||||
}
|
||||
)";
|
||||
|
||||
std::string starts_at_bottom_def;
|
||||
if (input_starts_at_bottom) {
|
||||
starts_at_bottom_def = R"(
|
||||
#define INPUT_STARTS_AT_BOTTOM
|
||||
)";
|
||||
}
|
||||
|
||||
// Create program and set parameters.
|
||||
const std::string extract_sub_rect_vertex_src =
|
||||
absl::StrCat(mediapipe::kMediaPipeVertexShaderPreamble,
|
||||
starts_at_bottom_def, kExtractSubRectVertexShader);
|
||||
const std::string extract_sub_rect_frag_src = absl::StrCat(
|
||||
mediapipe::kMediaPipeFragmentShaderPreamble, kExtractSubRectFragBody);
|
||||
mediapipe::GlhCreateProgram(extract_sub_rect_vertex_src.c_str(),
|
||||
extract_sub_rect_frag_src.c_str(),
|
||||
kNumAttributes, &attr_name[0], attr_location,
|
||||
&program_);
|
||||
|
||||
RET_CHECK(program_) << "Problem initializing image to tensor program.";
|
||||
glUseProgram(program_);
|
||||
glUniform1i(glGetUniformLocation(program_, "input_texture"), 1);
|
||||
alpha_id_ = glGetUniformLocation(program_, "alpha");
|
||||
beta_id_ = glGetUniformLocation(program_, "beta");
|
||||
matrix_id_ = glGetUniformLocation(program_, "transform_matrix");
|
||||
|
||||
glGenFramebuffers(1, &framebuffer_);
|
||||
|
||||
// vertex storage
|
||||
glGenBuffers(2, vbo_);
|
||||
glGenVertexArrays(1, &vao_);
|
||||
|
||||
// vbo 0
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]);
|
||||
glBufferData(GL_ARRAY_BUFFER, sizeof(mediapipe::kBasicSquareVertices),
|
||||
mediapipe::kBasicSquareVertices, GL_STATIC_DRAW);
|
||||
|
||||
// vbo 1
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]);
|
||||
glBufferData(GL_ARRAY_BUFFER, sizeof(mediapipe::kBasicTextureVertices),
|
||||
mediapipe::kBasicTextureVertices, GL_STATIC_DRAW);
|
||||
|
||||
glBindBuffer(GL_ARRAY_BUFFER, 0);
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
});
|
||||
}
|
||||
|
||||
Size GetImageSize(const Packet& image_packet) override {
|
||||
const auto& image = image_packet.Get<mediapipe::GpuBuffer>();
|
||||
return {image.width(), image.height()};
|
||||
}
|
||||
|
||||
::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
|
||||
const RotatedRect& roi,
|
||||
const Size& output_dims,
|
||||
float range_min,
|
||||
float range_max) override {
|
||||
const auto& input = image_packet.Get<mediapipe::GpuBuffer>();
|
||||
if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) {
|
||||
return InvalidArgumentError(
|
||||
absl::StrCat("Only BGRA/RGBA textures are supported, passed format: ",
|
||||
static_cast<uint32_t>(input.format())));
|
||||
}
|
||||
|
||||
constexpr int kNumChannels = 3;
|
||||
Tensor tensor(
|
||||
Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, output_dims.height, output_dims.width, kNumChannels});
|
||||
|
||||
MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext(
|
||||
[this, &tensor, &input, &roi, &output_dims, range_min,
|
||||
range_max]() -> ::mediapipe::Status {
|
||||
auto input_texture = gl_helper_.CreateSourceTexture(input);
|
||||
|
||||
constexpr float kInputImageRangeMin = 0.0f;
|
||||
constexpr float kInputImageRangeMax = 1.0f;
|
||||
ASSIGN_OR_RETURN(auto transform,
|
||||
GetValueRangeTransformation(kInputImageRangeMin,
|
||||
kInputImageRangeMax,
|
||||
range_min, range_max));
|
||||
auto tensor_view = tensor.GetOpenGlTexture2dWriteView();
|
||||
MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi,
|
||||
/*flip_horizontaly=*/false,
|
||||
transform.scale, transform.offset,
|
||||
output_dims, &tensor_view));
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
|
||||
return tensor;
|
||||
}
|
||||
|
||||
::mediapipe::Status ExtractSubRect(const mediapipe::GlTexture& texture,
|
||||
const RotatedRect& sub_rect,
|
||||
bool flip_horizontaly, float alpha,
|
||||
float beta, const Size& output_dims,
|
||||
Tensor::OpenGlTexture2dView* output) {
|
||||
std::array<float, 16> transform_mat;
|
||||
GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(),
|
||||
texture.height(), flip_horizontaly,
|
||||
&transform_mat);
|
||||
|
||||
glDisable(GL_DEPTH_TEST);
|
||||
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
|
||||
glViewport(0, 0, output_dims.width, output_dims.height);
|
||||
|
||||
glActiveTexture(GL_TEXTURE0);
|
||||
glBindTexture(GL_TEXTURE_2D, output->name());
|
||||
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
|
||||
output->name(), 0);
|
||||
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(texture.target(), texture.name());
|
||||
|
||||
ASSIGN_OR_RETURN(auto overrides, GlParametersOverride::Create(
|
||||
{{GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE},
|
||||
{GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE},
|
||||
{GL_TEXTURE_MIN_FILTER, GL_LINEAR},
|
||||
{GL_TEXTURE_MAG_FILTER, GL_LINEAR}}));
|
||||
|
||||
glUseProgram(program_);
|
||||
glUniform1f(alpha_id_, alpha);
|
||||
glUniform1f(beta_id_, beta);
|
||||
glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data());
|
||||
|
||||
// vao
|
||||
glBindVertexArray(vao_);
|
||||
|
||||
// vbo 0
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]);
|
||||
glEnableVertexAttribArray(kAttribVertex);
|
||||
glVertexAttribPointer(kAttribVertex, 2, GL_FLOAT, 0, 0, nullptr);
|
||||
|
||||
// vbo 1
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]);
|
||||
glEnableVertexAttribArray(kAttribTexturePosition);
|
||||
glVertexAttribPointer(kAttribTexturePosition, 2, GL_FLOAT, 0, 0, nullptr);
|
||||
|
||||
// draw
|
||||
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
|
||||
|
||||
// cleanup
|
||||
glDisableVertexAttribArray(kAttribVertex);
|
||||
glDisableVertexAttribArray(kAttribTexturePosition);
|
||||
glBindBuffer(GL_ARRAY_BUFFER, 0);
|
||||
glBindVertexArray(0);
|
||||
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(GL_TEXTURE_2D, 0);
|
||||
glActiveTexture(GL_TEXTURE0);
|
||||
glBindTexture(GL_TEXTURE_2D, 0);
|
||||
|
||||
return overrides.Revert();
|
||||
}
|
||||
|
||||
~GlProcessor() override {
|
||||
gl_helper_.RunInGlContext([this]() {
|
||||
// Release OpenGL resources.
|
||||
if (framebuffer_ != 0) glDeleteFramebuffers(1, &framebuffer_);
|
||||
if (program_ != 0) glDeleteProgram(program_);
|
||||
if (vao_ != 0) glDeleteVertexArrays(1, &vao_);
|
||||
glDeleteBuffers(2, vbo_);
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
mediapipe::GlCalculatorHelper gl_helper_;
|
||||
GLuint vao_ = 0;
|
||||
GLuint vbo_[2] = {0, 0};
|
||||
GLuint program_ = 0;
|
||||
GLuint framebuffer_ = 0;
|
||||
GLint alpha_id_ = 0;
|
||||
GLint beta_id_ = 0;
|
||||
GLint matrix_id_ = 0;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
|
||||
CreateImageToGlTextureTensorConverter(CalculatorContext* cc,
|
||||
bool input_starts_at_bottom) {
|
||||
auto result = absl::make_unique<GlProcessor>();
|
||||
MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom));
|
||||
|
||||
// Simply "return std::move(result)" failed to build on macOS with bazel.
|
||||
return std::unique_ptr<ImageToTensorConverter>(std::move(result));
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_
|
||||
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_
|
||||
|
||||
#include "mediapipe/framework/port.h"
|
||||
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Creates image to tensor (represented as OpenGL texture) converter.
|
||||
// NOTE: mediapipe::GlCalculatorHelper::UpdateContract invocation must precede
|
||||
// converter creation.
|
||||
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
|
||||
CreateImageToGlTextureTensorConverter(CalculatorContext* cc,
|
||||
bool input_starts_at_bottom);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_
|
||||
@@ -0,0 +1,397 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_metal.h"
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
|
||||
#import <Metal/Metal.h>
|
||||
|
||||
#include <array>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
#include "mediapipe/gpu/MPPMetalHelper.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "tensorflow/lite/delegates/gpu/common/shape.h"
|
||||
#include "tensorflow/lite/delegates/gpu/common/types.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
// clang-format off
|
||||
// a square formed by 2 triangles
|
||||
const float kBasicSquareVertices[] = {
|
||||
-1, 1, 0, 1,
|
||||
1, 1, 0, 1,
|
||||
1, -1, 0, 1,
|
||||
-1, 1, 0, 1,
|
||||
1, -1, 0, 1,
|
||||
-1, -1, 0, 1,
|
||||
};
|
||||
|
||||
// maps a texture to kBasicSquareVertices via aspect fill
|
||||
const float kBasicTextureVertices[] = {
|
||||
0, 0, 0, 1,
|
||||
1, 0, 0, 1,
|
||||
1, 1, 0, 1,
|
||||
0, 0, 0, 1,
|
||||
1, 1, 0, 1,
|
||||
0, 1, 0, 1,
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
constexpr char kShaderLibHeader[] = R"(
|
||||
#include <metal_stdlib>
|
||||
|
||||
using namespace metal;
|
||||
|
||||
struct TextureVertex
|
||||
{
|
||||
float4 position [[position]];
|
||||
float2 uv;
|
||||
};
|
||||
)";
|
||||
|
||||
constexpr char kVertexShader[] = R"(
|
||||
vertex TextureVertex vertexShader(
|
||||
constant float4 *position [[buffer(0)]],
|
||||
device float4* tex_coords [[buffer(1)]],
|
||||
constant float4x4& transform_matrix [[buffer(2)]],
|
||||
uint vid [[vertex_id]]) {
|
||||
TextureVertex vert;
|
||||
vert.position = position[vid];
|
||||
vert.uv = (tex_coords[vid] * transform_matrix).xy;
|
||||
return vert;
|
||||
}
|
||||
)";
|
||||
|
||||
constexpr char kFragmentShader[] = R"(
|
||||
#ifdef OUTPUT_F16C4
|
||||
#define Type4 half4
|
||||
#define Type half
|
||||
#endif // OUTPUT_F16C4
|
||||
|
||||
#ifdef OUTPUT_F32C4
|
||||
#define Type4 float4
|
||||
#define Type float
|
||||
#endif // OUTPUT_F32C4
|
||||
|
||||
fragment Type4 fragmentShader(TextureVertex vertex_output [[stage_in]],
|
||||
texture2d<Type> texture [[texture(0)]],
|
||||
constant float* parameters [[buffer(1)]])
|
||||
{
|
||||
const float alpha = parameters[0];
|
||||
const float beta = parameters[1];
|
||||
|
||||
constexpr sampler linear_sampler(address::clamp_to_edge, min_filter::linear,
|
||||
mag_filter::linear);
|
||||
|
||||
Type4 texture_pixel = texture.sample(linear_sampler, vertex_output.uv);
|
||||
return Type4(alpha * texture_pixel.rgb + beta, 0);
|
||||
}
|
||||
)";
|
||||
|
||||
enum class OutputFormat { kF16C4, kF32C4 };
|
||||
|
||||
MTLPixelFormat GetPixelFormat(OutputFormat output_format) {
|
||||
switch (output_format) {
|
||||
case OutputFormat::kF16C4:
|
||||
return MTLPixelFormatRGBA16Float;
|
||||
case OutputFormat::kF32C4:
|
||||
return MTLPixelFormatRGBA32Float;
|
||||
}
|
||||
}
|
||||
int GetBytesPerRaw(OutputFormat output_format, const tflite::gpu::HW& size) {
|
||||
std::size_t type_size;
|
||||
switch (output_format) {
|
||||
case OutputFormat::kF16C4:
|
||||
type_size = sizeof(tflite::gpu::HalfBits);
|
||||
break;
|
||||
case OutputFormat::kF32C4:
|
||||
type_size = sizeof(float);
|
||||
break;
|
||||
}
|
||||
constexpr int kNumChannels = 4;
|
||||
return size.w * kNumChannels * type_size;
|
||||
}
|
||||
|
||||
class SubRectExtractorMetal {
|
||||
public:
|
||||
static ::mediapipe::StatusOr<std::unique_ptr<SubRectExtractorMetal>> Make(
|
||||
id<MTLDevice> device, OutputFormat output_format) {
|
||||
id<MTLRenderPipelineState> pipeline_state;
|
||||
MP_RETURN_IF_ERROR(SubRectExtractorMetal::MakePipelineState(
|
||||
device, output_format, &pipeline_state));
|
||||
|
||||
return absl::make_unique<SubRectExtractorMetal>(device, pipeline_state,
|
||||
output_format);
|
||||
}
|
||||
|
||||
SubRectExtractorMetal(id<MTLDevice> device,
|
||||
id<MTLRenderPipelineState> pipeline_state,
|
||||
OutputFormat output_format)
|
||||
: device_(device),
|
||||
pipeline_state_(pipeline_state),
|
||||
output_format_(output_format) {
|
||||
positions_buffer_ =
|
||||
[device_ newBufferWithBytes:kBasicSquareVertices
|
||||
length:sizeof(kBasicSquareVertices)
|
||||
options:MTLResourceOptionCPUCacheModeDefault];
|
||||
|
||||
tex_coords_buffer_ =
|
||||
[device_ newBufferWithBytes:kBasicTextureVertices
|
||||
length:sizeof(kBasicTextureVertices)
|
||||
options:MTLResourceOptionCPUCacheModeDefault];
|
||||
|
||||
transform_mat_buffer_ =
|
||||
[device_ newBufferWithBytes:&transform_mat_
|
||||
length:sizeof(transform_mat_)
|
||||
options:MTLResourceOptionCPUCacheModeDefault];
|
||||
}
|
||||
|
||||
::mediapipe::Status Execute(id<MTLTexture> input_texture,
|
||||
const RotatedRect& sub_rect,
|
||||
bool flip_horizontaly, float alpha, float beta,
|
||||
const tflite::gpu::HW& destination_size,
|
||||
id<MTLCommandBuffer> command_buffer,
|
||||
id<MTLBuffer> destination) {
|
||||
auto output_texture = MTLTextureWithBuffer(destination_size, destination);
|
||||
return InternalExecute(input_texture, sub_rect, flip_horizontaly, alpha,
|
||||
beta, destination_size, command_buffer,
|
||||
output_texture);
|
||||
}
|
||||
|
||||
private:
|
||||
id<MTLTexture> MTLTextureWithBuffer(const tflite::gpu::HW& size,
|
||||
id<MTLBuffer> buffer) {
|
||||
MTLTextureDescriptor* texture_desc = [MTLTextureDescriptor
|
||||
texture2DDescriptorWithPixelFormat:GetPixelFormat(output_format_)
|
||||
width:size.w
|
||||
height:size.h
|
||||
mipmapped:NO];
|
||||
texture_desc.usage = MTLTextureUsageRenderTarget;
|
||||
|
||||
NSUInteger output_bytes_per_row = GetBytesPerRaw(output_format_, size);
|
||||
|
||||
id<MTLTexture> texture =
|
||||
[buffer newTextureWithDescriptor:texture_desc
|
||||
offset:0
|
||||
bytesPerRow:output_bytes_per_row];
|
||||
return texture;
|
||||
}
|
||||
|
||||
::mediapipe::Status InternalExecute(id<MTLTexture> input_texture,
|
||||
const RotatedRect& sub_rect,
|
||||
bool flip_horizontaly, float alpha,
|
||||
float beta,
|
||||
const tflite::gpu::HW& destination_size,
|
||||
id<MTLCommandBuffer> command_buffer,
|
||||
id<MTLTexture> output_texture) {
|
||||
RET_CHECK(command_buffer != nil);
|
||||
RET_CHECK(output_texture != nil);
|
||||
|
||||
// Obtain texture mapping coordinates transformation matrix and copy its
|
||||
// data to the buffer.
|
||||
GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width,
|
||||
input_texture.height,
|
||||
flip_horizontaly, &transform_mat_);
|
||||
std::memcpy(reinterpret_cast<float*>(transform_mat_buffer_.contents),
|
||||
transform_mat_.data(), sizeof(transform_mat_));
|
||||
|
||||
// Create parameters wrapper.
|
||||
float parameters[] = {alpha, beta};
|
||||
|
||||
// Now everything is ready to go!
|
||||
// Setup render pass.
|
||||
MTLRenderPassDescriptor* render_pass_desc =
|
||||
[MTLRenderPassDescriptor renderPassDescriptor];
|
||||
render_pass_desc.colorAttachments[0].texture = output_texture;
|
||||
render_pass_desc.colorAttachments[0].storeAction = MTLStoreActionStore;
|
||||
render_pass_desc.colorAttachments[0].loadAction = MTLLoadActionClear;
|
||||
|
||||
// Setup render command encoder.
|
||||
id<MTLRenderCommandEncoder> command_encoder =
|
||||
[command_buffer renderCommandEncoderWithDescriptor:render_pass_desc];
|
||||
[command_encoder setRenderPipelineState:pipeline_state_];
|
||||
[command_encoder setVertexBuffer:positions_buffer_ offset:0 atIndex:0];
|
||||
[command_encoder setVertexBuffer:tex_coords_buffer_ offset:0 atIndex:1];
|
||||
[command_encoder setVertexBuffer:transform_mat_buffer_ offset:0 atIndex:2];
|
||||
[command_encoder setFragmentTexture:input_texture atIndex:0];
|
||||
[command_encoder setFragmentBytes:¶meters
|
||||
length:sizeof(parameters)
|
||||
atIndex:1];
|
||||
|
||||
[command_encoder drawPrimitives:MTLPrimitiveTypeTriangle
|
||||
vertexStart:0
|
||||
vertexCount:6];
|
||||
[command_encoder endEncoding];
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
static ::mediapipe::Status MakePipelineState(
|
||||
id<MTLDevice> device, OutputFormat output_format,
|
||||
id<MTLRenderPipelineState>* pipeline_state) {
|
||||
RET_CHECK(pipeline_state != nil);
|
||||
|
||||
std::string output_type_def;
|
||||
MTLPixelFormat pixel_format;
|
||||
switch (output_format) {
|
||||
case OutputFormat::kF16C4:
|
||||
output_type_def = R"(
|
||||
#define OUTPUT_F16C4
|
||||
)";
|
||||
break;
|
||||
case OutputFormat::kF32C4:
|
||||
output_type_def = R"(
|
||||
#define OUTPUT_F32C4
|
||||
)";
|
||||
break;
|
||||
}
|
||||
|
||||
std::string shader_lib = absl::StrCat(kShaderLibHeader, output_type_def,
|
||||
kVertexShader, kFragmentShader);
|
||||
NSError* error = nil;
|
||||
NSString* library_source =
|
||||
[NSString stringWithUTF8String:shader_lib.c_str()];
|
||||
|
||||
id<MTLLibrary> library =
|
||||
[device newLibraryWithSource:library_source options:nil error:&error];
|
||||
RET_CHECK(library != nil) << "Couldn't create a shader library"
|
||||
<< [[error localizedDescription] UTF8String];
|
||||
|
||||
id<MTLFunction> vertex_function =
|
||||
[library newFunctionWithName:@"vertexShader"];
|
||||
RET_CHECK(vertex_function != nil)
|
||||
<< "Failed creating a new vertex function!";
|
||||
|
||||
id<MTLFunction> fragment_function =
|
||||
[library newFunctionWithName:@"fragmentShader"];
|
||||
RET_CHECK(fragment_function != nil)
|
||||
<< "Failed creating a new fragment function!";
|
||||
|
||||
MTLRenderPipelineDescriptor* pipelineDescriptor =
|
||||
[MTLRenderPipelineDescriptor new];
|
||||
pipelineDescriptor.vertexFunction = vertex_function;
|
||||
pipelineDescriptor.fragmentFunction = fragment_function;
|
||||
pipelineDescriptor.colorAttachments[0].pixelFormat =
|
||||
GetPixelFormat(output_format);
|
||||
|
||||
*pipeline_state =
|
||||
[device newRenderPipelineStateWithDescriptor:pipelineDescriptor
|
||||
error:&error];
|
||||
RET_CHECK(error == nil) << "Couldn't create a pipeline state"
|
||||
<< [[error localizedDescription] UTF8String];
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
id<MTLBuffer> positions_buffer_;
|
||||
id<MTLBuffer> tex_coords_buffer_;
|
||||
id<MTLBuffer> transform_mat_buffer_;
|
||||
id<MTLDevice> device_;
|
||||
id<MTLRenderPipelineState> pipeline_state_;
|
||||
std::array<float, 16> transform_mat_;
|
||||
OutputFormat output_format_;
|
||||
};
|
||||
|
||||
class MetalProcessor : public ImageToTensorConverter {
|
||||
public:
|
||||
::mediapipe::Status Init(CalculatorContext* cc) {
|
||||
metal_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
|
||||
RET_CHECK(metal_helper_);
|
||||
ASSIGN_OR_RETURN(extractor_,
|
||||
SubRectExtractorMetal::Make(metal_helper_.mtlDevice,
|
||||
OutputFormat::kF32C4));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Size GetImageSize(const Packet& image_packet) override {
|
||||
const auto& image = image_packet.Get<mediapipe::GpuBuffer>();
|
||||
return {image.width(), image.height()};
|
||||
}
|
||||
|
||||
::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
|
||||
const RotatedRect& roi,
|
||||
const Size& output_dims,
|
||||
float range_min,
|
||||
float range_max) override {
|
||||
const auto& input = image_packet.Get<mediapipe::GpuBuffer>();
|
||||
if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) {
|
||||
return InvalidArgumentError(
|
||||
absl::StrCat("Only BGRA/RGBA textures are supported, passed "
|
||||
"format: ",
|
||||
static_cast<uint32_t>(input.format())));
|
||||
}
|
||||
|
||||
@autoreleasepool {
|
||||
id<MTLTexture> texture = [metal_helper_ metalTextureWithGpuBuffer:input];
|
||||
|
||||
constexpr int kNumChannels = 4;
|
||||
Tensor tensor(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, output_dims.height, output_dims.width,
|
||||
kNumChannels});
|
||||
|
||||
constexpr float kInputImageRangeMin = 0.0f;
|
||||
constexpr float kInputImageRangeMax = 1.0f;
|
||||
ASSIGN_OR_RETURN(
|
||||
auto transform,
|
||||
GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
|
||||
range_min, range_max));
|
||||
|
||||
id<MTLCommandBuffer> command_buffer = [metal_helper_ commandBuffer];
|
||||
const auto& buffer_view = tensor.GetMtlBufferWriteView(command_buffer);
|
||||
MP_RETURN_IF_ERROR(extractor_->Execute(
|
||||
texture, roi,
|
||||
/*flip_horizontaly=*/false, transform.scale, transform.offset,
|
||||
tflite::gpu::HW(output_dims.height, output_dims.width),
|
||||
command_buffer, buffer_view.buffer()));
|
||||
[command_buffer commit];
|
||||
// TODO: consider removing waitUntilCompleted
|
||||
[command_buffer waitUntilCompleted];
|
||||
|
||||
return tensor;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
MPPMetalHelper* metal_helper_ = nil;
|
||||
std::unique_ptr<SubRectExtractorMetal> extractor_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
|
||||
CreateMetalConverter(CalculatorContext* cc) {
|
||||
auto result = absl::make_unique<MetalProcessor>();
|
||||
MP_RETURN_IF_ERROR(result->Init(cc));
|
||||
|
||||
// Simply "return std::move(result)" failed to build on macOS with bazel.
|
||||
return std::unique_ptr<ImageToTensorConverter>(std::move(result));
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_METAL_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_METAL_H_
|
||||
|
||||
#include "mediapipe/framework/port.h"
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Creates Metal image-to-tensor converter.
|
||||
// NOTE: [MPPMetalHelper updateContract:...] invocation must precede
|
||||
// converter creation.
|
||||
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
|
||||
CreateMetalConverter(CalculatorContext* cc);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_METAL_H_
|
||||
@@ -0,0 +1,116 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
class OpenCvProcessor : public ImageToTensorConverter {
|
||||
public:
|
||||
Size GetImageSize(const Packet& image_packet) override {
|
||||
const auto& image = image_packet.Get<mediapipe::ImageFrame>();
|
||||
return {image.Width(), image.Height()};
|
||||
}
|
||||
|
||||
::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
|
||||
const RotatedRect& roi,
|
||||
const Size& output_dims,
|
||||
float range_min,
|
||||
float range_max) override {
|
||||
const auto& input = image_packet.Get<mediapipe::ImageFrame>();
|
||||
if (input.Format() != mediapipe::ImageFormat::SRGB &&
|
||||
input.Format() != mediapipe::ImageFormat::SRGBA) {
|
||||
return InvalidArgumentError(
|
||||
absl::StrCat("Only RGBA/RGB formats are supported, passed format: ",
|
||||
static_cast<uint32_t>(input.Format())));
|
||||
}
|
||||
cv::Mat src = mediapipe::formats::MatView(&input);
|
||||
|
||||
constexpr int kNumChannels = 3;
|
||||
Tensor tensor(
|
||||
Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, output_dims.height, output_dims.width, kNumChannels});
|
||||
auto buffer_view = tensor.GetCpuWriteView();
|
||||
cv::Mat dst(output_dims.height, output_dims.width, CV_32FC3,
|
||||
buffer_view.buffer<float>());
|
||||
|
||||
const cv::RotatedRect rotated_rect(cv::Point2f(roi.center_x, roi.center_y),
|
||||
cv::Size2f(roi.width, roi.height),
|
||||
roi.rotation * 180.f / M_PI);
|
||||
cv::Mat src_points;
|
||||
cv::boxPoints(rotated_rect, src_points);
|
||||
|
||||
const float dst_width = output_dims.width;
|
||||
const float dst_height = output_dims.height;
|
||||
/* clang-format off */
|
||||
float dst_corners[8] = {0.0f, dst_height,
|
||||
0.0f, 0.0f,
|
||||
dst_width, 0.0f,
|
||||
dst_width, dst_height};
|
||||
/* clang-format on */
|
||||
|
||||
cv::Mat dst_points = cv::Mat(4, 2, CV_32F, dst_corners);
|
||||
cv::Mat projection_matrix =
|
||||
cv::getPerspectiveTransform(src_points, dst_points);
|
||||
cv::Mat transformed;
|
||||
cv::warpPerspective(src, transformed, projection_matrix,
|
||||
cv::Size(dst_width, dst_height),
|
||||
/*flags=*/cv::INTER_LINEAR,
|
||||
/*borderMode=*/cv::BORDER_REPLICATE);
|
||||
|
||||
if (transformed.channels() > kNumChannels) {
|
||||
cv::Mat proper_channels_mat;
|
||||
cv::cvtColor(transformed, proper_channels_mat, cv::COLOR_RGBA2RGB);
|
||||
transformed = proper_channels_mat;
|
||||
}
|
||||
|
||||
constexpr float kInputImageRangeMin = 0.0f;
|
||||
constexpr float kInputImageRangeMax = 255.0f;
|
||||
ASSIGN_OR_RETURN(
|
||||
auto transform,
|
||||
GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
|
||||
range_min, range_max));
|
||||
transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset);
|
||||
return tensor;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
|
||||
CreateOpenCvConverter(CalculatorContext* cc) {
|
||||
// Simply "return absl::make_unique<OpenCvProcessor>()" failed to build on
|
||||
// macOS with bazel.
|
||||
return std::unique_ptr<ImageToTensorConverter>(
|
||||
absl::make_unique<OpenCvProcessor>());
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,32 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_OPENCV_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_OPENCV_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Creates OpenCV image-to-tensor converter.
|
||||
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
|
||||
CreateOpenCvConverter(CalculatorContext* cc);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_OPENCV_H_
|
||||
@@ -0,0 +1,176 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
|
||||
#include <array>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
RotatedRect GetRoi(int input_width, int input_height,
|
||||
absl::optional<mediapipe::NormalizedRect> norm_rect) {
|
||||
if (norm_rect) {
|
||||
return {.center_x = norm_rect->x_center() * input_width,
|
||||
.center_y = norm_rect->y_center() * input_height,
|
||||
.width = norm_rect->width() * input_width,
|
||||
.height = norm_rect->height() * input_height,
|
||||
.rotation = norm_rect->rotation()};
|
||||
}
|
||||
return {.center_x = 0.5f * input_width,
|
||||
.center_y = 0.5f * input_height,
|
||||
.width = static_cast<float>(input_width),
|
||||
.height = static_cast<float>(input_height),
|
||||
.rotation = 0};
|
||||
}
|
||||
|
||||
::mediapipe::StatusOr<std::array<float, 4>> PadRoi(int input_tensor_width,
|
||||
int input_tensor_height,
|
||||
bool keep_aspect_ratio,
|
||||
RotatedRect* roi) {
|
||||
if (!keep_aspect_ratio) {
|
||||
return std::array<float, 4>{0.0f, 0.0f, 0.0f, 0.0f};
|
||||
}
|
||||
|
||||
RET_CHECK(input_tensor_width > 0 && input_tensor_height > 0)
|
||||
<< "Input tensor width and height must be > 0.";
|
||||
const float tensor_aspect_ratio =
|
||||
static_cast<float>(input_tensor_height) / input_tensor_width;
|
||||
|
||||
RET_CHECK(roi->width > 0 && roi->height > 0)
|
||||
<< "ROI width and height must be > 0.";
|
||||
const float roi_aspect_ratio = roi->height / roi->width;
|
||||
|
||||
float vertical_padding = 0.0f;
|
||||
float horizontal_padding = 0.0f;
|
||||
float new_width;
|
||||
float new_height;
|
||||
if (tensor_aspect_ratio > roi_aspect_ratio) {
|
||||
new_width = roi->width;
|
||||
new_height = roi->width * tensor_aspect_ratio;
|
||||
vertical_padding = (1.0f - roi_aspect_ratio / tensor_aspect_ratio) / 2.0f;
|
||||
} else {
|
||||
new_width = roi->height / tensor_aspect_ratio;
|
||||
new_height = roi->height;
|
||||
horizontal_padding = (1.0f - tensor_aspect_ratio / roi_aspect_ratio) / 2.0f;
|
||||
}
|
||||
|
||||
roi->width = new_width;
|
||||
roi->height = new_height;
|
||||
|
||||
return std::array<float, 4>{horizontal_padding, vertical_padding,
|
||||
horizontal_padding, vertical_padding};
|
||||
}
|
||||
|
||||
::mediapipe::StatusOr<ValueTransformation> GetValueRangeTransformation(
|
||||
float from_range_min, float from_range_max, float to_range_min,
|
||||
float to_range_max) {
|
||||
RET_CHECK_LT(from_range_min, from_range_max)
|
||||
<< "Invalid FROM range: min >= max.";
|
||||
RET_CHECK_LT(to_range_min, to_range_max) << "Invalid TO range: min >= max.";
|
||||
const float scale =
|
||||
(to_range_max - to_range_min) / (from_range_max - from_range_min);
|
||||
const float offset = to_range_min - from_range_min * scale;
|
||||
return ValueTransformation{scale, offset};
|
||||
}
|
||||
|
||||
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
||||
int rect_width, int rect_height,
|
||||
bool flip_horizontaly,
|
||||
std::array<float, 16>* matrix_ptr) {
|
||||
std::array<float, 16>& matrix = *matrix_ptr;
|
||||
// The resulting matrix is multiplication of below commented out matrices:
|
||||
// post_scale_matrix
|
||||
// * translate_matrix
|
||||
// * rotate_matrix
|
||||
// * flip_matrix
|
||||
// * scale_matrix
|
||||
// * initial_translate_matrix
|
||||
|
||||
// Matrix to convert X,Y to [-0.5, 0.5] range "initial_translate_matrix"
|
||||
// { 1.0f, 0.0f, 0.0f, -0.5f}
|
||||
// { 0.0f, 1.0f, 0.0f, -0.5f}
|
||||
// { 0.0f, 0.0f, 1.0f, 0.0f}
|
||||
// { 0.0f, 0.0f, 0.0f, 1.0f}
|
||||
|
||||
const float a = sub_rect.width;
|
||||
const float b = sub_rect.height;
|
||||
// Matrix to scale X,Y,Z to sub rect "scale_matrix"
|
||||
// Z has the same scale as X.
|
||||
// { a, 0.0f, 0.0f, 0.0f}
|
||||
// {0.0f, b, 0.0f, 0.0f}
|
||||
// {0.0f, 0.0f, a, 0.0f}
|
||||
// {0.0f, 0.0f, 0.0f, 1.0f}
|
||||
|
||||
const float flip = flip_horizontaly ? -1 : 1;
|
||||
// Matrix for optional horizontal flip around middle of output image.
|
||||
// { fl , 0.0f, 0.0f, 0.0f}
|
||||
// { 0.0f, 1.0f, 0.0f, 0.0f}
|
||||
// { 0.0f, 0.0f, 1.0f, 0.0f}
|
||||
// { 0.0f, 0.0f, 0.0f, 1.0f}
|
||||
|
||||
const float c = std::cos(sub_rect.rotation);
|
||||
const float d = std::sin(sub_rect.rotation);
|
||||
// Matrix to do rotation around Z axis "rotate_matrix"
|
||||
// { c, -d, 0.0f, 0.0f}
|
||||
// { d, c, 0.0f, 0.0f}
|
||||
// { 0.0f, 0.0f, 1.0f, 0.0f}
|
||||
// { 0.0f, 0.0f, 0.0f, 1.0f}
|
||||
|
||||
const float e = sub_rect.center_x;
|
||||
const float f = sub_rect.center_y;
|
||||
// Matrix to do X,Y translation of sub rect within parent rect
|
||||
// "translate_matrix"
|
||||
// {1.0f, 0.0f, 0.0f, e }
|
||||
// {0.0f, 1.0f, 0.0f, f }
|
||||
// {0.0f, 0.0f, 1.0f, 0.0f}
|
||||
// {0.0f, 0.0f, 0.0f, 1.0f}
|
||||
|
||||
const float g = 1.0f / rect_width;
|
||||
const float h = 1.0f / rect_height;
|
||||
// Matrix to scale X,Y,Z to [0.0, 1.0] range "post_scale_matrix"
|
||||
// {g, 0.0f, 0.0f, 0.0f}
|
||||
// {0.0f, h, 0.0f, 0.0f}
|
||||
// {0.0f, 0.0f, g, 0.0f}
|
||||
// {0.0f, 0.0f, 0.0f, 1.0f}
|
||||
|
||||
// row 1
|
||||
matrix[0] = a * c * flip * g;
|
||||
matrix[1] = -b * d * g;
|
||||
matrix[2] = 0.0f;
|
||||
matrix[3] = (-0.5f * a * c * flip + 0.5f * b * d + e) * g;
|
||||
|
||||
// row 2
|
||||
matrix[4] = a * d * flip * h;
|
||||
matrix[5] = b * c * h;
|
||||
matrix[6] = 0.0f;
|
||||
matrix[7] = (-0.5f * b * c - 0.5f * a * d * flip + f) * h;
|
||||
|
||||
// row 3
|
||||
matrix[8] = 0.0f;
|
||||
matrix[9] = 0.0f;
|
||||
matrix[10] = a * g;
|
||||
matrix[11] = 0.0f;
|
||||
|
||||
// row 4
|
||||
matrix[12] = 0.0f;
|
||||
matrix[13] = 0.0f;
|
||||
matrix[14] = 0.0f;
|
||||
matrix[15] = 1.0f;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_UTILS_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_UTILS_H_
|
||||
|
||||
#include <array>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
struct RotatedRect {
|
||||
float center_x;
|
||||
float center_y;
|
||||
float width;
|
||||
float height;
|
||||
float rotation;
|
||||
};
|
||||
|
||||
// Generates a new ROI or converts it from normalized rect.
|
||||
RotatedRect GetRoi(int input_width, int input_height,
|
||||
absl::optional<mediapipe::NormalizedRect> norm_rect);
|
||||
|
||||
// Pads ROI, so extraction happens correctly if aspect ratio is to be kept.
|
||||
// Returns letterbox padding applied.
|
||||
::mediapipe::StatusOr<std::array<float, 4>> PadRoi(int input_tensor_width,
|
||||
int input_tensor_height,
|
||||
bool keep_aspect_ratio,
|
||||
RotatedRect* roi);
|
||||
|
||||
// Represents a transformation of value which involves scaling and offsetting.
|
||||
// To apply transformation:
|
||||
// ValueTransformation transform = ...
|
||||
// float transformed_value = transform.scale * value + transfrom.offset;
|
||||
struct ValueTransformation {
|
||||
float scale;
|
||||
float offset;
|
||||
};
|
||||
|
||||
// Returns value transformation to apply to a value in order to convert it from
|
||||
// [from_range_min, from_range_max] into [to_range_min, to_range_max] range.
|
||||
// from_range_min must be less than from_range_max
|
||||
// to_range_min must be less than to_range_max
|
||||
::mediapipe::StatusOr<ValueTransformation> GetValueRangeTransformation(
|
||||
float from_range_min, float from_range_max, float to_range_min,
|
||||
float to_range_max);
|
||||
|
||||
// Populates 4x4 "matrix" with row major order transformation matrix which
|
||||
// maps (x, y) in range [0, 1] (describing points of @sub_rect)
|
||||
// to (x', y') in range [0, 1]*** (describing points of a rect:
|
||||
// [0, @rect_width] x [0, @rect_height] = RECT).
|
||||
//
|
||||
// *** (x', y') will go out of the range for points from @sub_rect
|
||||
// which are not contained by RECT and it's expected behavior
|
||||
//
|
||||
// @sub_rect - rotated sub rect in absolute coordinates
|
||||
// @rect_width - rect width
|
||||
// @rect_height - rect height
|
||||
// @flip_horizontaly - we need to flip the output buffer.
|
||||
// @matrix - 4x4 matrix (array of 16 elements) to populate
|
||||
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
||||
int rect_width, int rect_height,
|
||||
bool flip_horizontaly,
|
||||
std::array<float, 16>* matrix);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_UTILS_H_
|
||||
@@ -0,0 +1,161 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
using ::testing::ElementsAre;
|
||||
using ::testing::ElementsAreArray;
|
||||
|
||||
testing::Matcher<RotatedRect> EqRotatedRect(float width, float height,
|
||||
float center_x, float center_y,
|
||||
float rotation) {
|
||||
return testing::AllOf(
|
||||
testing::Field(&RotatedRect::width, testing::FloatEq(width)),
|
||||
testing::Field(&RotatedRect::height, testing::FloatEq(height)),
|
||||
testing::Field(&RotatedRect::center_x, testing::FloatEq(center_x)),
|
||||
testing::Field(&RotatedRect::center_y, testing::FloatEq(center_y)),
|
||||
testing::Field(&RotatedRect::rotation, testing::FloatEq(rotation)));
|
||||
}
|
||||
|
||||
TEST(GetRoi, NoNormRect) {
|
||||
EXPECT_THAT(GetRoi(4, 4, {}), EqRotatedRect(4, 4, 2, 2, 0));
|
||||
EXPECT_THAT(GetRoi(25, 15, {}), EqRotatedRect(25, 15, 12.5f, 7.5f, 0));
|
||||
}
|
||||
|
||||
TEST(GetRoi, WholeImageNormRect) {
|
||||
mediapipe::NormalizedRect norm_rect;
|
||||
norm_rect.set_width(1.0f);
|
||||
norm_rect.set_height(1.0f);
|
||||
norm_rect.set_x_center(0.5f);
|
||||
norm_rect.set_y_center(0.5f);
|
||||
norm_rect.set_rotation(0.0f);
|
||||
EXPECT_THAT(GetRoi(4, 4, norm_rect), EqRotatedRect(4, 4, 2, 2, 0));
|
||||
EXPECT_THAT(GetRoi(25, 15, norm_rect), EqRotatedRect(25, 15, 12.5f, 7.5f, 0));
|
||||
}
|
||||
|
||||
TEST(GetRoi, ExpandedNormRect) {
|
||||
mediapipe::NormalizedRect norm_rect;
|
||||
norm_rect.set_width(4.0f);
|
||||
norm_rect.set_height(2.0f);
|
||||
norm_rect.set_x_center(0.5f);
|
||||
norm_rect.set_y_center(1.0f);
|
||||
norm_rect.set_rotation(3.0f);
|
||||
EXPECT_THAT(GetRoi(4, 4, norm_rect), EqRotatedRect(16, 8, 2, 4, 3));
|
||||
EXPECT_THAT(GetRoi(25, 15, norm_rect), EqRotatedRect(100, 30, 12.5f, 15, 3));
|
||||
}
|
||||
|
||||
TEST(PadRoi, NoPadding) {
|
||||
RotatedRect roi{.center_x = 20,
|
||||
.center_y = 10,
|
||||
.width = 100,
|
||||
.height = 200,
|
||||
.rotation = 5};
|
||||
auto status_or_value = PadRoi(10, 10, /*keep_aspect_ratio=*/false, &roi);
|
||||
MP_ASSERT_OK(status_or_value);
|
||||
EXPECT_THAT(status_or_value.ValueOrDie(),
|
||||
ElementsAreArray({0.0f, 0.0f, 0.0f, 0.0f}));
|
||||
EXPECT_THAT(roi, EqRotatedRect(100, 200, 20, 10, 5));
|
||||
}
|
||||
|
||||
TEST(PadRoi, HorizontalPadding) {
|
||||
RotatedRect roi{.center_x = 20,
|
||||
.center_y = 10,
|
||||
.width = 100,
|
||||
.height = 200,
|
||||
.rotation = 5};
|
||||
auto status_or_value = PadRoi(10, 10, /*keep_aspect_ratio=*/true, &roi);
|
||||
MP_ASSERT_OK(status_or_value);
|
||||
EXPECT_THAT(status_or_value.ValueOrDie(),
|
||||
ElementsAreArray({0.25f, 0.0f, 0.25f, 0.0f}));
|
||||
EXPECT_THAT(roi, EqRotatedRect(200, 200, 20, 10, 5));
|
||||
}
|
||||
|
||||
TEST(PadRoi, VerticalPadding) {
|
||||
RotatedRect roi{
|
||||
.center_x = 1, .center_y = 2, .width = 21, .height = 19, .rotation = 3};
|
||||
const float expected_horizontal_padding = (21 - 19) / 2.0f / 21;
|
||||
auto status_or_value = PadRoi(10, 10, /*keep_aspect_ratio=*/true, &roi);
|
||||
MP_ASSERT_OK(status_or_value);
|
||||
EXPECT_THAT(
|
||||
status_or_value.ValueOrDie(),
|
||||
ElementsAre(testing::FloatEq(0.0f),
|
||||
testing::FloatNear(expected_horizontal_padding, 1e-6),
|
||||
testing::FloatEq(0.0f),
|
||||
testing::FloatNear(expected_horizontal_padding, 1e-6)));
|
||||
EXPECT_THAT(roi, EqRotatedRect(21, 21, 1, 2, 3));
|
||||
}
|
||||
|
||||
testing::Matcher<ValueTransformation> EqValueTransformation(float scale,
|
||||
float offset) {
|
||||
return ::testing::AllOf(
|
||||
testing::Field(&ValueTransformation::scale, testing::FloatEq(scale)),
|
||||
testing::Field(&ValueTransformation::offset, testing::FloatEq(offset)));
|
||||
}
|
||||
|
||||
TEST(GetValueRangeTransformation, PixelToFloatZeroCenter) {
|
||||
auto status_or_value = GetValueRangeTransformation(
|
||||
/*from_range_min=*/0.0f, /*from_range_max=*/255.0f,
|
||||
/*to_range_min=*/-1.0f, /*to_range_max=*/1.0f);
|
||||
MP_ASSERT_OK(status_or_value);
|
||||
EXPECT_THAT(status_or_value.ValueOrDie(),
|
||||
EqValueTransformation(/*scale=*/2 / 255.0f,
|
||||
/*offset=*/-1.0f));
|
||||
}
|
||||
|
||||
TEST(GetValueRangeTransformation, PixelToFloat) {
|
||||
auto status_or_value = GetValueRangeTransformation(
|
||||
/*from_range_min=*/0.0f, /*from_range_max=*/255.0f,
|
||||
/*to_range_min=*/0.0f, /*to_range_max=*/1.0f);
|
||||
MP_ASSERT_OK(status_or_value);
|
||||
EXPECT_THAT(status_or_value.ValueOrDie(),
|
||||
EqValueTransformation(/*scale=*/1 / 255.0f,
|
||||
/*offset=*/0.0f));
|
||||
}
|
||||
|
||||
TEST(GetValueRangeTransformation, FloatToFloatNoOp) {
|
||||
auto status_or_value = GetValueRangeTransformation(
|
||||
/*from_range_min=*/0.0f, /*from_range_max=*/1.0f,
|
||||
/*to_range_min=*/0.0f, /*to_range_max=*/1.0f);
|
||||
MP_ASSERT_OK(status_or_value);
|
||||
EXPECT_THAT(status_or_value.ValueOrDie(),
|
||||
EqValueTransformation(/*scale=*/1.0f, /*offset=*/0.0f));
|
||||
}
|
||||
|
||||
TEST(GetValueRangeTransformation, PixelToPixelNoOp) {
|
||||
auto status_or_value = GetValueRangeTransformation(
|
||||
/*from_range_min=*/0.0f, /*from_range_max=*/255.0f,
|
||||
/*to_range_min=*/0.0f, /*to_range_max=*/255.0f);
|
||||
MP_ASSERT_OK(status_or_value);
|
||||
EXPECT_THAT(status_or_value.ValueOrDie(),
|
||||
EqValueTransformation(/*scale=*/1.0f, /*offset=*/0.0f));
|
||||
}
|
||||
|
||||
TEST(GetValueRangeTransformation, FloatToPixel) {
|
||||
auto status_or_value = GetValueRangeTransformation(
|
||||
/*from_range_min=*/0.0f, /*from_range_max=*/1.0f,
|
||||
/*to_range_min=*/0.0f, /*to_range_max=*/255.0f);
|
||||
MP_ASSERT_OK(status_or_value);
|
||||
EXPECT_THAT(status_or_value.ValueOrDie(),
|
||||
EqValueTransformation(/*scale=*/255.0f, /*offset=*/0.0f));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,832 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/util/tflite/config.h"
|
||||
|
||||
#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)
|
||||
#include "mediapipe/util/cpu_util.h"
|
||||
#endif // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__
|
||||
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
#include "tensorflow/lite/error_reporter.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
#include "tensorflow/lite/kernels/register.h"
|
||||
#include "tensorflow/lite/model.h"
|
||||
|
||||
#if defined(MEDIAPIPE_ANDROID)
|
||||
#include "mediapipe/util/android/file/base/file.h"
|
||||
#include "mediapipe/util/android/file/base/filesystem.h"
|
||||
#include "mediapipe/util/android/file/base/helpers.h"
|
||||
#endif // ANDROID
|
||||
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "mediapipe/util/tflite/tflite_gpu_runner.h"
|
||||
#include "tensorflow/lite/delegates/gpu/common/shape.h"
|
||||
#include "tensorflow/lite/delegates/gpu/gl_delegate.h"
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
|
||||
#if MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
#import <Metal/Metal.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
|
||||
#import "mediapipe/gpu/MPPMetalHelper.h"
|
||||
#include "mediapipe/gpu/MPPMetalUtil.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "tensorflow/lite/delegates/gpu/common/shape.h"
|
||||
#include "tensorflow/lite/delegates/gpu/metal/buffer_convert.h"
|
||||
#include "tensorflow/lite/delegates/gpu/metal_delegate.h"
|
||||
#include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h"
|
||||
#endif // MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
|
||||
#if !defined(MEDIAPIPE_EDGE_TPU)
|
||||
#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h"
|
||||
#endif // !EDGETPU
|
||||
#if defined(MEDIAPIPE_ANDROID)
|
||||
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
|
||||
#endif // ANDROID
|
||||
|
||||
namespace {
|
||||
// Commonly used to compute the number of blocks to launch in a kernel.
|
||||
int NumGroups(const int size, const int group_size) { // NOLINT
|
||||
return (size + group_size - 1) / group_size;
|
||||
}
|
||||
|
||||
// Round up n to next multiple of m.
|
||||
template <typename T>
|
||||
T RoundUp(T n, T m) {
|
||||
return ((n + m - T{1}) / m) * m;
|
||||
}
|
||||
|
||||
bool ShouldUseGpu(const mediapipe::InferenceCalculatorOptions& options) {
|
||||
return (
|
||||
!options.has_delegate() || // Use GPU delegate if delegate not specified
|
||||
(options.has_delegate() && options.delegate().has_gpu()));
|
||||
}
|
||||
|
||||
constexpr char kTensorsTag[] = "TENSORS";
|
||||
} // namespace
|
||||
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
#include "edgetpu.h"
|
||||
|
||||
// Creates and returns an Edge TPU interpreter to run the given edgetpu model.
|
||||
std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
|
||||
const tflite::FlatBufferModel& model,
|
||||
tflite::ops::builtin::BuiltinOpResolver* resolver,
|
||||
edgetpu::EdgeTpuContext* edgetpu_context) {
|
||||
resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp());
|
||||
std::unique_ptr<tflite::Interpreter> interpreter;
|
||||
if (tflite::InterpreterBuilder(model, *resolver)(&interpreter) != kTfLiteOk) {
|
||||
std::cerr << "Failed to build edge TPU interpreter." << std::endl;
|
||||
}
|
||||
interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context);
|
||||
interpreter->SetNumThreads(1);
|
||||
if (interpreter->AllocateTensors() != kTfLiteOk) {
|
||||
std::cerr << "Failed to allocate edge TPU tensors." << std::endl;
|
||||
}
|
||||
return interpreter;
|
||||
}
|
||||
#endif // MEDIAPIPE_EDGE_TPU
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
#if MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
namespace {
|
||||
tflite::gpu::BHWC BhwcFromTensorShape(const Tensor::Shape& shape) {
|
||||
tflite::gpu::BHWC result;
|
||||
result.b = shape.dims[0];
|
||||
switch (shape.dims.size()) {
|
||||
case 1:
|
||||
// result.b is already filled.
|
||||
break;
|
||||
case 2:
|
||||
result.h = 1;
|
||||
result.w = 1;
|
||||
result.c = shape.dims[1];
|
||||
break;
|
||||
case 3:
|
||||
result.h = 1;
|
||||
result.w = shape.dims[1];
|
||||
result.c = shape.dims[2];
|
||||
break;
|
||||
case 4:
|
||||
result.h = shape.dims[1];
|
||||
result.w = shape.dims[2];
|
||||
result.c = shape.dims[3];
|
||||
break;
|
||||
default:
|
||||
// Handles 0 and >4.
|
||||
LOG(FATAL)
|
||||
<< "Dimensions size must be in range [1,4] for GPU inference, but "
|
||||
<< shape.dims.size() << " is provided";
|
||||
}
|
||||
return result;
|
||||
}
|
||||
} // namespace
|
||||
#endif // MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
|
||||
// Returns number of threads to configure XNNPACK delegate with.
|
||||
// (Equal to user provided value if specified. Otherwise, it returns number of
|
||||
// high cores (hard-coded to 1 for Emscripten without Threads extension))
|
||||
int GetXnnpackNumThreads(const mediapipe::InferenceCalculatorOptions& opts) {
|
||||
static constexpr int kDefaultNumThreads = -1;
|
||||
if (opts.has_delegate() && opts.delegate().has_xnnpack() &&
|
||||
opts.delegate().xnnpack().num_threads() != kDefaultNumThreads) {
|
||||
return opts.delegate().xnnpack().num_threads();
|
||||
}
|
||||
#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)
|
||||
return InferHigherCoreIds().size();
|
||||
#else
|
||||
return 1;
|
||||
#endif // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__
|
||||
}
|
||||
|
||||
// Calculator Header Section
|
||||
|
||||
// Runs inference on the provided input Tensors and TFLite model.
|
||||
//
|
||||
// Creates an interpreter with given model and calls invoke().
|
||||
// Optionally run inference on CPU/GPU.
|
||||
//
|
||||
// This calculator can be used with TensorConverterCalculator to get the
|
||||
// appropriate inputs.
|
||||
//
|
||||
// When the input tensors are on CPU, gpu inference is optional and can be
|
||||
// specified in the calculator options.
|
||||
// When the input tensors are on GPU, inference is GPU and output can be CPU or
|
||||
// GPU.
|
||||
//
|
||||
// Input:
|
||||
// TENSORS - Vector of Tensors
|
||||
//
|
||||
// Output:
|
||||
// TENSORS - Vector of Tensors
|
||||
//
|
||||
// Input side packet:
|
||||
// CUSTOM_OP_RESOLVER (optional) - Use a custom op resolver,
|
||||
// instead of the builtin one.
|
||||
// MODEL (optional) - Use to specify TfLite model
|
||||
// (std::unique_ptr<tflite::FlatBufferModel,
|
||||
// std::function<void(tflite::FlatBufferModel*)>>)
|
||||
//
|
||||
// Example use:
|
||||
// node {
|
||||
// calculator: "InferenceCalculator"
|
||||
// input_stream: "TENSORS:tensor_image"
|
||||
// output_stream: "TENSORS:tensors"
|
||||
// options: {
|
||||
// [mediapipe.InferenceCalculatorOptions.ext] {
|
||||
// model_path: "modelname.tflite"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// or
|
||||
//
|
||||
// node {
|
||||
// calculator: "InferenceCalculator"
|
||||
// input_stream: "TENSORS:tensor_image"
|
||||
// input_side_packet: "MODEL:model"
|
||||
// output_stream: "TENSORS:tensors"
|
||||
// options: {
|
||||
// [mediapipe.InferenceCalculatorOptions.ext] {
|
||||
// model_path: "modelname.tflite"
|
||||
// delegate { gpu {} }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// IMPORTANT Notes:
|
||||
// Tensors are assumed to be ordered correctly (sequentially added to model).
|
||||
// Input tensors are assumed to be of the correct size and already normalized.
|
||||
|
||||
class InferenceCalculator : public CalculatorBase {
|
||||
public:
|
||||
using TfLiteDelegatePtr =
|
||||
std::unique_ptr<TfLiteDelegate, std::function<void(TfLiteDelegate*)>>;
|
||||
using TfLiteModelPtr =
|
||||
std::unique_ptr<tflite::FlatBufferModel,
|
||||
std::function<void(tflite::FlatBufferModel*)>>;
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
::mediapipe::Status ReadKernelsFromFile();
|
||||
::mediapipe::Status WriteKernelsToFile();
|
||||
::mediapipe::Status LoadModel(CalculatorContext* cc);
|
||||
::mediapipe::StatusOr<Packet> GetModelAsPacket(const CalculatorContext& cc);
|
||||
::mediapipe::Status LoadDelegate(CalculatorContext* cc);
|
||||
::mediapipe::Status InitTFLiteGPURunner(CalculatorContext* cc);
|
||||
|
||||
Packet model_packet_;
|
||||
std::unique_ptr<tflite::Interpreter> interpreter_;
|
||||
TfLiteDelegatePtr delegate_;
|
||||
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
mediapipe::GlCalculatorHelper gpu_helper_;
|
||||
std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
|
||||
#elif MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
MPPMetalHelper* gpu_helper_ = nullptr;
|
||||
TFLBufferConvert* converter_to_BPHWC4_ = nil;
|
||||
TFLBufferConvert* converter_from_BPHWC4_ = nil;
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
|
||||
#if MEDIAPIPE_TFLITE_GPU_SUPPORTED
|
||||
std::vector<Tensor::Shape> output_shapes_;
|
||||
std::vector<std::unique_ptr<Tensor>> gpu_buffers_in_;
|
||||
std::vector<std::unique_ptr<Tensor>> gpu_buffers_out_;
|
||||
#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED
|
||||
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
std::shared_ptr<edgetpu::EdgeTpuContext> edgetpu_context_ =
|
||||
edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice();
|
||||
#endif
|
||||
|
||||
bool use_advanced_gpu_api_ = false;
|
||||
bool use_gpu_delegate_ = false;
|
||||
|
||||
bool use_kernel_caching_ = false;
|
||||
std::string cached_kernel_filename_;
|
||||
};
|
||||
REGISTER_CALCULATOR(InferenceCalculator);
|
||||
|
||||
::mediapipe::Status InferenceCalculator::GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kTensorsTag));
|
||||
cc->Inputs().Tag(kTensorsTag).Set<std::vector<Tensor>>();
|
||||
RET_CHECK(cc->Outputs().HasTag(kTensorsTag));
|
||||
cc->Outputs().Tag(kTensorsTag).Set<std::vector<Tensor>>();
|
||||
|
||||
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
|
||||
RET_CHECK(!options.model_path().empty() ^
|
||||
cc->InputSidePackets().HasTag("MODEL"))
|
||||
<< "Either model as side packet or model path in options is required.";
|
||||
|
||||
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
|
||||
cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Set<tflite::ops::builtin::BuiltinOpResolver>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("MODEL")) {
|
||||
cc->InputSidePackets().Tag("MODEL").Set<TfLiteModelPtr>();
|
||||
}
|
||||
|
||||
if (ShouldUseGpu(options)) {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
|
||||
#elif MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
|
||||
#endif
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status InferenceCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE || MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
|
||||
if (ShouldUseGpu(options)) {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
use_advanced_gpu_api_ = options.has_delegate() &&
|
||||
options.delegate().has_gpu() &&
|
||||
options.delegate().gpu().use_advanced_gpu_api();
|
||||
use_kernel_caching_ =
|
||||
use_advanced_gpu_api_ && options.delegate().gpu().use_kernel_caching();
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
use_gpu_delegate_ = !use_advanced_gpu_api_;
|
||||
}
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE || MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
|
||||
if (use_kernel_caching_) {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE && defined(MEDIAPIPE_ANDROID)
|
||||
cached_kernel_filename_ =
|
||||
"/sdcard/" + mediapipe::File::Basename(options.model_path()) + ".ker";
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE && MEDIAPIPE_ANDROID
|
||||
}
|
||||
|
||||
// When use_advanced_gpu_api_, model loading is handled in InitTFLiteGPURunner
|
||||
// for everything.
|
||||
if (!use_advanced_gpu_api_) {
|
||||
MP_RETURN_IF_ERROR(LoadModel(cc));
|
||||
}
|
||||
|
||||
if (use_gpu_delegate_ || use_advanced_gpu_api_) {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
|
||||
MP_RETURN_IF_ERROR(
|
||||
gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
|
||||
return use_advanced_gpu_api_ ? InitTFLiteGPURunner(cc)
|
||||
: LoadDelegate(cc);
|
||||
}));
|
||||
#elif MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
|
||||
RET_CHECK(gpu_helper_);
|
||||
MP_RETURN_IF_ERROR(LoadDelegate(cc));
|
||||
#endif
|
||||
} else {
|
||||
MP_RETURN_IF_ERROR(LoadDelegate(cc));
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status InferenceCalculator::Process(CalculatorContext* cc) {
|
||||
if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
const auto& input_tensors =
|
||||
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
|
||||
RET_CHECK(!input_tensors.empty());
|
||||
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
|
||||
|
||||
if (use_gpu_delegate_ || use_advanced_gpu_api_) {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
if (use_advanced_gpu_api_) {
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
|
||||
[this, &input_tensors, &output_tensors]() -> ::mediapipe::Status {
|
||||
for (int i = 0; i < input_tensors.size(); ++i) {
|
||||
MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor(
|
||||
input_tensors[i].GetOpenGlBufferReadView().name(), i));
|
||||
}
|
||||
output_tensors->reserve(output_shapes_.size());
|
||||
for (int i = 0; i < output_shapes_.size(); ++i) {
|
||||
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
|
||||
output_shapes_[i]);
|
||||
MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToOutputTensor(
|
||||
output_tensors->back().GetOpenGlBufferWriteView().name(), i));
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
} else {
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
|
||||
[this, &input_tensors]() -> ::mediapipe::Status {
|
||||
// Explicitly copy input.
|
||||
for (int i = 0; i < input_tensors.size(); ++i) {
|
||||
glBindBuffer(GL_COPY_READ_BUFFER,
|
||||
input_tensors[i].GetOpenGlBufferReadView().name());
|
||||
glBindBuffer(
|
||||
GL_COPY_WRITE_BUFFER,
|
||||
gpu_buffers_in_[i]->GetOpenGlBufferWriteView().name());
|
||||
glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0,
|
||||
0, input_tensors[i].bytes());
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
}
|
||||
#elif MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
// Explicit copy input with conversion float 32 bits to 16 bits.
|
||||
id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
|
||||
command_buffer.label = @"InferenceCalculatorConvert";
|
||||
id<MTLComputeCommandEncoder> compute_encoder =
|
||||
[command_buffer computeCommandEncoder];
|
||||
for (int i = 0; i < input_tensors.size(); ++i) {
|
||||
auto input_view = input_tensors[i].GetMtlBufferReadView(command_buffer);
|
||||
// Reshape tensor.
|
||||
tflite::gpu::BHWC shape = BhwcFromTensorShape(input_tensors[i].shape());
|
||||
auto gpu_buffer_view =
|
||||
gpu_buffers_in_[i]->GetMtlBufferWriteView(command_buffer);
|
||||
[converter_to_BPHWC4_ convertWithEncoder:compute_encoder
|
||||
shape:shape
|
||||
sourceBuffer:input_view.buffer()
|
||||
convertedBuffer:gpu_buffer_view.buffer()];
|
||||
}
|
||||
[compute_encoder endEncoding];
|
||||
[command_buffer commit];
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
} else {
|
||||
// Read CPU input into tensors.
|
||||
for (int i = 0; i < input_tensors.size(); ++i) {
|
||||
const Tensor* input_tensor = &input_tensors[i];
|
||||
auto input_tensor_view = input_tensor->GetCpuReadView();
|
||||
auto input_tensor_buffer = input_tensor_view.buffer<float>();
|
||||
float* local_tensor_buffer = interpreter_->typed_input_tensor<float>(i);
|
||||
std::memcpy(local_tensor_buffer, input_tensor_buffer,
|
||||
input_tensor->bytes());
|
||||
}
|
||||
}
|
||||
|
||||
// Run inference.
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
if (use_advanced_gpu_api_) {
|
||||
RET_CHECK(tflite_gpu_runner_->Invoke().ok());
|
||||
} else {
|
||||
RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
|
||||
}
|
||||
#else
|
||||
RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
|
||||
if (use_gpu_delegate_ || use_advanced_gpu_api_) {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
if (use_gpu_delegate_) {
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
|
||||
[this, &output_tensors]() -> ::mediapipe::Status {
|
||||
output_tensors->reserve(output_shapes_.size());
|
||||
for (int i = 0; i < output_shapes_.size(); ++i) {
|
||||
const auto& t = gpu_buffers_out_[i];
|
||||
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
|
||||
gpu_buffers_out_[i]->shape());
|
||||
auto read_view = t->GetOpenGlBufferReadView();
|
||||
glBindBuffer(GL_COPY_READ_BUFFER, read_view.name());
|
||||
auto write_view =
|
||||
output_tensors->back().GetOpenGlBufferWriteView();
|
||||
glBindBuffer(GL_COPY_WRITE_BUFFER, write_view.name());
|
||||
glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0,
|
||||
0, t->bytes());
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
}
|
||||
// Output tensors are already bound if use_advanced_gpu_api_ is true.
|
||||
#elif MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
|
||||
command_buffer.label = @"InferenceBPHWC4Convert";
|
||||
id<MTLComputeCommandEncoder> convert_command =
|
||||
[command_buffer computeCommandEncoder];
|
||||
output_tensors->reserve(output_shapes_.size());
|
||||
for (int i = 0; i < output_shapes_.size(); ++i) {
|
||||
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
|
||||
output_shapes_[i]);
|
||||
// Reshape tensor.
|
||||
tflite::gpu::BHWC shape = BhwcFromTensorShape(output_shapes_[i]);
|
||||
auto read_view =
|
||||
gpu_buffers_out_[i]->GetMtlBufferReadView(command_buffer);
|
||||
auto write_view =
|
||||
output_tensors->at(i).GetMtlBufferWriteView(command_buffer);
|
||||
[converter_from_BPHWC4_ convertWithEncoder:convert_command
|
||||
shape:shape
|
||||
sourceBuffer:read_view.buffer()
|
||||
convertedBuffer:write_view.buffer()];
|
||||
}
|
||||
[convert_command endEncoding];
|
||||
[command_buffer commit];
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
} else {
|
||||
// Output result tensors (CPU).
|
||||
const auto& tensor_indexes = interpreter_->outputs();
|
||||
output_tensors->reserve(tensor_indexes.size());
|
||||
for (int i = 0; i < tensor_indexes.size(); ++i) {
|
||||
TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]);
|
||||
output_tensors->emplace_back(
|
||||
Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{std::vector<int>{
|
||||
tensor->dims->data, tensor->dims->data + tensor->dims->size}});
|
||||
auto cpu_view = output_tensors->back().GetCpuWriteView();
|
||||
std::memcpy(cpu_view.buffer<float>(), tensor->data.f,
|
||||
output_tensors->back().bytes());
|
||||
}
|
||||
}
|
||||
cc->Outputs()
|
||||
.Tag(kTensorsTag)
|
||||
.Add(output_tensors.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status InferenceCalculator::WriteKernelsToFile() {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE && defined(MEDIAPIPE_ANDROID)
|
||||
if (use_kernel_caching_) {
|
||||
// Save kernel file.
|
||||
auto kernel_cache = absl::make_unique<std::vector<uint8_t>>(
|
||||
tflite_gpu_runner_->GetSerializedBinaryCache());
|
||||
std::string cache_str(kernel_cache->begin(), kernel_cache->end());
|
||||
MP_RETURN_IF_ERROR(
|
||||
mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
|
||||
}
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE && MEDIAPIPE_ANDROID
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status InferenceCalculator::Close(CalculatorContext* cc) {
|
||||
MP_RETURN_IF_ERROR(WriteKernelsToFile());
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
if (use_gpu_delegate_) {
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
|
||||
gpu_buffers_in_.clear();
|
||||
gpu_buffers_out_.clear();
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
}
|
||||
#elif MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
converter_to_BPHWC4_ = nil;
|
||||
converter_from_BPHWC4_ = nil;
|
||||
gpu_buffers_in_.clear();
|
||||
gpu_buffers_out_.clear();
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
edgetpu_context_.reset();
|
||||
#endif
|
||||
interpreter_ = nullptr;
|
||||
delegate_ = nullptr;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status InferenceCalculator::ReadKernelsFromFile() {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE && defined(MEDIAPIPE_ANDROID)
|
||||
if (use_kernel_caching_) {
|
||||
// Load pre-compiled kernel file.
|
||||
if (mediapipe::File::Exists(cached_kernel_filename_)) {
|
||||
std::string cache_str;
|
||||
MP_RETURN_IF_ERROR(
|
||||
mediapipe::file::GetContents(cached_kernel_filename_, &cache_str));
|
||||
std::vector<uint8_t> cache_vec(cache_str.begin(), cache_str.end());
|
||||
tflite_gpu_runner_->SetSerializedBinaryCache(std::move(cache_vec));
|
||||
}
|
||||
}
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE && MEDIAPIPE_ANDROID
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status InferenceCalculator::InitTFLiteGPURunner(
|
||||
CalculatorContext* cc) {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc));
|
||||
const auto& model = *model_packet_.Get<TfLiteModelPtr>();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver;
|
||||
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
|
||||
op_resolver = cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Get<tflite::ops::builtin::BuiltinOpResolver>();
|
||||
}
|
||||
|
||||
// Create runner
|
||||
tflite::gpu::InferenceOptions options;
|
||||
options.priority1 = tflite::gpu::InferencePriority::MIN_LATENCY;
|
||||
options.priority2 = tflite::gpu::InferencePriority::AUTO;
|
||||
options.priority3 = tflite::gpu::InferencePriority::AUTO;
|
||||
options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
|
||||
tflite_gpu_runner_ = std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
|
||||
MP_RETURN_IF_ERROR(
|
||||
tflite_gpu_runner_->InitializeWithModel(model, op_resolver));
|
||||
|
||||
// Create and bind OpenGL buffers for outputs.
|
||||
// The buffers are created once and their ids are passed to calculator outputs
|
||||
output_shapes_.resize(tflite_gpu_runner_->outputs_size());
|
||||
for (int i = 0; i < tflite_gpu_runner_->outputs_size(); ++i) {
|
||||
output_shapes_[i] = {tflite_gpu_runner_->GetOutputShapes()[i].b,
|
||||
tflite_gpu_runner_->GetOutputShapes()[i].h,
|
||||
tflite_gpu_runner_->GetOutputShapes()[i].w,
|
||||
tflite_gpu_runner_->GetOutputShapes()[i].c};
|
||||
}
|
||||
|
||||
MP_RETURN_IF_ERROR(ReadKernelsFromFile());
|
||||
|
||||
MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build());
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status InferenceCalculator::LoadModel(CalculatorContext* cc) {
|
||||
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc));
|
||||
const auto& model = *model_packet_.Get<TfLiteModelPtr>();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver;
|
||||
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
|
||||
op_resolver = cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Get<tflite::ops::builtin::BuiltinOpResolver>();
|
||||
}
|
||||
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
interpreter_ =
|
||||
BuildEdgeTpuInterpreter(model, &op_resolver, edgetpu_context_.get());
|
||||
#else
|
||||
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
|
||||
#endif // MEDIAPIPE_EDGE_TPU
|
||||
RET_CHECK(interpreter_);
|
||||
|
||||
#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_EDGE_TPU)
|
||||
interpreter_->SetNumThreads(1);
|
||||
#else
|
||||
interpreter_->SetNumThreads(
|
||||
cc->Options<mediapipe::InferenceCalculatorOptions>().cpu_num_thread());
|
||||
#endif // __EMSCRIPTEN__
|
||||
|
||||
RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
|
||||
// TODO: Support quantized tensors.
|
||||
CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type !=
|
||||
kTfLiteAffineQuantization);
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::StatusOr<Packet> InferenceCalculator::GetModelAsPacket(
|
||||
const CalculatorContext& cc) {
|
||||
const auto& options = cc.Options<mediapipe::InferenceCalculatorOptions>();
|
||||
if (!options.model_path().empty()) {
|
||||
std::string model_path = options.model_path();
|
||||
|
||||
ASSIGN_OR_RETURN(model_path, mediapipe::PathToResourceAsFile(model_path));
|
||||
|
||||
auto model = tflite::FlatBufferModel::BuildFromFile(model_path.c_str());
|
||||
RET_CHECK(model) << "Failed to load model from path.";
|
||||
return MakePacket<TfLiteModelPtr>(TfLiteModelPtr(
|
||||
model.release(), [](tflite::FlatBufferModel* model) { delete model; }));
|
||||
}
|
||||
if (cc.InputSidePackets().HasTag("MODEL")) {
|
||||
return cc.InputSidePackets().Tag("MODEL");
|
||||
}
|
||||
return ::mediapipe::Status(
|
||||
::mediapipe::StatusCode::kNotFound,
|
||||
"Must specify TFLite model as path or loaded model.");
|
||||
}
|
||||
|
||||
::mediapipe::Status InferenceCalculator::LoadDelegate(CalculatorContext* cc) {
|
||||
const auto& calculator_opts =
|
||||
cc->Options<mediapipe::InferenceCalculatorOptions>();
|
||||
if (calculator_opts.has_delegate() &&
|
||||
calculator_opts.delegate().has_tflite()) {
|
||||
// Default tflite inference requeqsted - no need to modify graph.
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
if (!use_gpu_delegate_) {
|
||||
#if defined(MEDIAPIPE_ANDROID)
|
||||
const bool nnapi_requested = calculator_opts.has_delegate()
|
||||
? calculator_opts.delegate().has_nnapi()
|
||||
: calculator_opts.use_nnapi();
|
||||
if (nnapi_requested) {
|
||||
// Attempt to use NNAPI.
|
||||
// If not supported, the default CPU delegate will be created and used.
|
||||
interpreter_->SetAllowFp16PrecisionForFp32(1);
|
||||
delegate_ =
|
||||
TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
|
||||
// No need to free according to tflite::NnApiDelegate()
|
||||
// documentation.
|
||||
});
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
|
||||
kTfLiteOk);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
#endif // MEDIAPIPE_ANDROID
|
||||
|
||||
#if defined(__EMSCRIPTEN__)
|
||||
const bool xnnpack_requested = true;
|
||||
#else
|
||||
const bool xnnpack_requested = calculator_opts.has_delegate() &&
|
||||
calculator_opts.delegate().has_xnnpack();
|
||||
#endif // __EMSCRIPTEN__
|
||||
|
||||
#if !defined(MEDIAPIPE_EDGE_TPU)
|
||||
if (xnnpack_requested) {
|
||||
TfLiteXNNPackDelegateOptions xnnpack_opts{};
|
||||
xnnpack_opts.num_threads = GetXnnpackNumThreads(calculator_opts);
|
||||
delegate_ = TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts),
|
||||
&TfLiteXNNPackDelegateDelete);
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
|
||||
kTfLiteOk);
|
||||
}
|
||||
#endif // !EDGETPU
|
||||
|
||||
// Return, no need for GPU delegate below.
|
||||
return ::mediapipe::OkStatus();
|
||||
} else {
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
// Configure and create the delegate.
|
||||
TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault();
|
||||
options.compile_options.precision_loss_allowed = 1;
|
||||
options.compile_options.preferred_gl_object_type =
|
||||
TFLITE_GL_OBJECT_TYPE_FASTEST;
|
||||
options.compile_options.dynamic_batch_enabled = 0;
|
||||
options.compile_options.inline_parameters = 1;
|
||||
delegate_ = TfLiteDelegatePtr(TfLiteGpuDelegateCreate(&options),
|
||||
&TfLiteGpuDelegateDelete);
|
||||
|
||||
// Get input image sizes.
|
||||
const auto& input_indices = interpreter_->inputs();
|
||||
for (int i = 0; i < input_indices.size(); ++i) {
|
||||
const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
|
||||
gpu_buffers_in_.emplace_back(absl::make_unique<Tensor>(
|
||||
Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{std::vector<int>{
|
||||
tensor->dims->data, tensor->dims->data + tensor->dims->size}}));
|
||||
RET_CHECK_EQ(
|
||||
TfLiteGpuDelegateBindBufferToTensor(
|
||||
delegate_.get(),
|
||||
gpu_buffers_in_.back()->GetOpenGlBufferWriteView().name(),
|
||||
interpreter_->inputs()[i]),
|
||||
kTfLiteOk);
|
||||
}
|
||||
interpreter_->SetAllowBufferHandleOutput(true);
|
||||
// Get output image sizes.
|
||||
const auto& output_indices = interpreter_->outputs();
|
||||
output_shapes_.resize(output_indices.size());
|
||||
// Create and bind output buffers.
|
||||
for (int i = 0; i < output_shapes_.size(); ++i) {
|
||||
const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]);
|
||||
gpu_buffers_out_.emplace_back(absl::make_unique<Tensor>(
|
||||
Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{std::vector<int>{
|
||||
tensor->dims->data, tensor->dims->data + tensor->dims->size}}));
|
||||
RET_CHECK_EQ(
|
||||
TfLiteGpuDelegateBindBufferToTensor(
|
||||
delegate_.get(),
|
||||
gpu_buffers_out_.back()->GetOpenGlBufferWriteView().name(),
|
||||
output_indices[i]),
|
||||
kTfLiteOk);
|
||||
}
|
||||
|
||||
// Must call this last.
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
|
||||
kTfLiteOk);
|
||||
#elif MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
// Configure and create the delegate.
|
||||
TFLGpuDelegateOptions options;
|
||||
options.allow_precision_loss = true;
|
||||
options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
|
||||
delegate_ = TfLiteDelegatePtr(TFLGpuDelegateCreate(&options),
|
||||
&TFLGpuDelegateDelete);
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
|
||||
kTfLiteOk);
|
||||
id<MTLDevice> device = gpu_helper_.mtlDevice;
|
||||
|
||||
// Get input image sizes.
|
||||
const auto& input_indices = interpreter_->inputs();
|
||||
for (int i = 0; i < input_indices.size(); ++i) {
|
||||
const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
|
||||
// Create and bind input buffer.
|
||||
std::vector<int> dims{tensor->dims->data,
|
||||
tensor->dims->data + tensor->dims->size};
|
||||
dims.back() = RoundUp(dims.back(), 4);
|
||||
gpu_buffers_in_.emplace_back(absl::make_unique<Tensor>(
|
||||
Tensor::ElementType::kFloat16, Tensor::Shape{dims}));
|
||||
auto buffer_view =
|
||||
gpu_buffers_in_[i]->GetMtlBufferWriteView(gpu_helper_.mtlDevice);
|
||||
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
|
||||
delegate_.get(), input_indices[i], buffer_view.buffer()),
|
||||
true);
|
||||
}
|
||||
|
||||
interpreter_->SetAllowBufferHandleOutput(true);
|
||||
// Get output image sizes.
|
||||
const auto& output_indices = interpreter_->outputs();
|
||||
output_shapes_.resize(output_indices.size());
|
||||
for (int i = 0; i < output_shapes_.size(); ++i) {
|
||||
const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]);
|
||||
RET_CHECK(tensor->dims->size <= 4);
|
||||
// Create and bind output buffers.
|
||||
// Channels are always padded to multiple of 4.
|
||||
std::vector<int> dims{tensor->dims->data,
|
||||
tensor->dims->data + tensor->dims->size};
|
||||
output_shapes_[i] = {dims};
|
||||
dims.back() = RoundUp(dims.back(), 4);
|
||||
gpu_buffers_out_.emplace_back(absl::make_unique<Tensor>(
|
||||
Tensor::ElementType::kFloat16, Tensor::Shape{dims}));
|
||||
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
|
||||
delegate_.get(), output_indices[i],
|
||||
gpu_buffers_out_[i]
|
||||
->GetMtlBufferWriteView(gpu_helper_.mtlDevice)
|
||||
.buffer()),
|
||||
true);
|
||||
}
|
||||
|
||||
// Create converter for GPU input.
|
||||
converter_to_BPHWC4_ = [[TFLBufferConvert alloc] initWithDevice:device
|
||||
isFloat16:true
|
||||
convertToPBHWC4:true];
|
||||
if (converter_to_BPHWC4_ == nil) {
|
||||
return mediapipe::InternalError(
|
||||
"Error initializating input buffer converter");
|
||||
}
|
||||
// Create converter for GPU output.
|
||||
converter_from_BPHWC4_ = [[TFLBufferConvert alloc] initWithDevice:device
|
||||
isFloat16:true
|
||||
convertToPBHWC4:false];
|
||||
if (converter_from_BPHWC4_ == nil) {
|
||||
return mediapipe::InternalError(
|
||||
"Error initializating output buffer converter");
|
||||
}
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,111 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
// Full Example:
|
||||
//
|
||||
// node {
|
||||
// calculator: "InferenceCalculator"
|
||||
// input_stream: "TENSOR_IN:image_tensors"
|
||||
// output_stream: "TENSOR_OUT:result_tensors"
|
||||
// options {
|
||||
// [mediapipe.InferenceCalculatorOptions.ext] {
|
||||
// model_path: "model.tflite"
|
||||
// delegate { gpu {} }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
message InferenceCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional InferenceCalculatorOptions ext = 336783863;
|
||||
}
|
||||
|
||||
message Delegate {
|
||||
// Default inference provided by tflite.
|
||||
message TfLite {}
|
||||
// Delegate to run GPU inference depending on the device.
|
||||
// (Can use OpenGl, OpenCl, Metal depending on the device.)
|
||||
message Gpu {
|
||||
// Experimental, Android/Linux only. Use TFLite GPU delegate API2 for
|
||||
// the NN inference.
|
||||
// example:
|
||||
// delegate: { gpu { use_advanced_gpu_api: true } }
|
||||
optional bool use_advanced_gpu_api = 1 [default = false];
|
||||
|
||||
// This option is valid for TFLite GPU delegate API2 only,
|
||||
// Choose any of available APIs to force running inference using it.
|
||||
enum API {
|
||||
ANY = 0;
|
||||
OPENGL = 1;
|
||||
OPENCL = 2;
|
||||
}
|
||||
optional API api = 4 [default = ANY];
|
||||
|
||||
// This option is valid for TFLite GPU delegate API2 only,
|
||||
// Set to true to use 16-bit float precision. If max precision is needed,
|
||||
// set to false for 32-bit float calculations only.
|
||||
optional bool allow_precision_loss = 3 [default = true];
|
||||
|
||||
// Load pre-compiled serialized binary cache to accelerate init process.
|
||||
// Only available for OpenCL delegate on Android.
|
||||
optional bool use_kernel_caching = 2 [default = false];
|
||||
}
|
||||
// Android only.
|
||||
message Nnapi {}
|
||||
message Xnnpack {
|
||||
// Number of threads for XNNPACK delegate. (By default, calculator tries
|
||||
// to choose optimal number of threads depending on the device.)
|
||||
optional int32 num_threads = 1 [default = -1];
|
||||
}
|
||||
|
||||
oneof delegate {
|
||||
TfLite tflite = 1;
|
||||
Gpu gpu = 2;
|
||||
Nnapi nnapi = 3;
|
||||
Xnnpack xnnpack = 4;
|
||||
}
|
||||
}
|
||||
|
||||
// Path to the TF Lite model (ex: /path/to/modelname.tflite).
|
||||
// On mobile, this is generally just modelname.tflite.
|
||||
optional string model_path = 1;
|
||||
|
||||
// Whether the TF Lite GPU or CPU backend should be used. Effective only when
|
||||
// input tensors are on CPU. For input tensors on GPU, GPU backend is always
|
||||
// used.
|
||||
// DEPRECATED: configure "delegate" instead.
|
||||
optional bool use_gpu = 2 [deprecated = true, default = false];
|
||||
|
||||
// Android only. When true, an NNAPI delegate will be used for inference.
|
||||
// If NNAPI is not available, then the default CPU delegate will be used
|
||||
// automatically.
|
||||
// DEPRECATED: configure "delegate" instead.
|
||||
optional bool use_nnapi = 3 [deprecated = true, default = false];
|
||||
|
||||
// The number of threads available to the interpreter. Effective only when
|
||||
// input tensors are on CPU and 'use_gpu' is false.
|
||||
optional int32 cpu_num_thread = 4 [default = -1];
|
||||
|
||||
// TfLite delegate to run inference.
|
||||
// NOTE: calculator is free to choose delegate if not specified explicitly.
|
||||
// NOTE: use_gpu/use_nnapi are ignored if specified. (Delegate takes
|
||||
// precedence over use_* deprecated options.)
|
||||
optional Delegate delegate = 5;
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_replace.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
#include "tensorflow/lite/error_reporter.h"
|
||||
#include "tensorflow/lite/kernels/register.h"
|
||||
#include "tensorflow/lite/model.h"
|
||||
|
||||
#ifdef __APPLE__
|
||||
#include <CoreFoundation/CoreFoundation.h>
|
||||
#endif // defined(__APPLE__)
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
using ::tflite::Interpreter;
|
||||
|
||||
void DoSmokeTest(const std::string& graph_proto) {
|
||||
const int width = 8;
|
||||
const int height = 8;
|
||||
const int channels = 3;
|
||||
// Prepare input tensor.
|
||||
auto input_vec = absl::make_unique<std::vector<Tensor>>();
|
||||
input_vec->emplace_back(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels});
|
||||
{
|
||||
auto view1 = input_vec->back().GetCpuWriteView();
|
||||
auto tensor_buffer = view1.buffer<float>();
|
||||
ASSERT_NE(tensor_buffer, nullptr);
|
||||
for (int i = 0; i < width * height * channels - 1; i++) {
|
||||
tensor_buffer[i] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare single calculator graph to and wait for packets.
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(graph_proto);
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor_out", &graph_config, &output_packets);
|
||||
CalculatorGraph graph(graph_config);
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
// Push the tensor into the graph.
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tensor_in", Adopt(input_vec.release()).At(Timestamp(0))));
|
||||
// Wait until the calculator done processing.
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
|
||||
// Get and process results.
|
||||
const std::vector<Tensor>& result_vec =
|
||||
output_packets[0].Get<std::vector<Tensor>>();
|
||||
ASSERT_EQ(1, result_vec.size());
|
||||
|
||||
const Tensor& result = result_vec[0];
|
||||
auto view = result.GetCpuReadView();
|
||||
auto result_buffer = view.buffer<float>();
|
||||
ASSERT_NE(result_buffer, nullptr);
|
||||
for (int i = 0; i < width * height * channels - 1; i++) {
|
||||
ASSERT_EQ(3, result_buffer[i]);
|
||||
}
|
||||
|
||||
// Fully close graph at end, otherwise calculator+tensors are destroyed
|
||||
// after calling WaitUntilDone().
|
||||
MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
// Tests a simple add model that adds an input tensor to itself.
|
||||
TEST(InferenceCalculatorTest, SmokeTest) {
|
||||
std::string graph_proto = R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:tensor_in"
|
||||
output_stream: "TENSORS:tensor_out"
|
||||
options {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/calculators/tensor/testdata/add.bin"
|
||||
$delegate
|
||||
}
|
||||
}
|
||||
}
|
||||
)";
|
||||
// Test CPU inference only.
|
||||
DoSmokeTest(/*graph_proto=*/absl::StrReplaceAll(
|
||||
graph_proto, {{"$delegate", "delegate { tflite {} }"}}));
|
||||
DoSmokeTest(/*graph_proto=*/absl::StrReplaceAll(
|
||||
graph_proto, {{"$delegate", "delegate { xnnpack {} }"}}));
|
||||
DoSmokeTest(/*graph_proto=*/absl::StrReplaceAll(
|
||||
graph_proto,
|
||||
{{"$delegate", "delegate { xnnpack { num_threads: 10 } }"}}));
|
||||
}
|
||||
|
||||
TEST(InferenceCalculatorTest, SmokeTest_ModelAsInputSidePacket) {
|
||||
std::string graph_proto = R"(
|
||||
input_stream: "tensor_in"
|
||||
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:model_path"
|
||||
options: {
|
||||
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
|
||||
packet { string_value: "mediapipe/calculators/tensor/testdata/add.bin" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "LocalFileContentsCalculator"
|
||||
input_side_packet: "FILE_PATH:model_path"
|
||||
output_side_packet: "CONTENTS:model_blob"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "TfLiteModelCalculator"
|
||||
input_side_packet: "MODEL_BLOB:model_blob"
|
||||
output_side_packet: "MODEL:model"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:tensor_in"
|
||||
output_stream: "TENSORS:tensor_out"
|
||||
input_side_packet: "MODEL:model"
|
||||
options {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
delegate { tflite {} }
|
||||
}
|
||||
}
|
||||
}
|
||||
)";
|
||||
DoSmokeTest(graph_proto);
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,676 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
#import <Metal/Metal.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
|
||||
#import "mediapipe/gpu/MPPMetalHelper.h"
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION < MEDIAPIPE_OPENGL_ES_31
|
||||
#include "mediapipe/gpu/gl_simple_shaders.h"
|
||||
#include "mediapipe/gpu/shader_util.h"
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION < MEDIAPIPE_OPENGL_ES_31
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
namespace {
|
||||
constexpr int kWorkgroupSize = 8; // Block size for GPU shader.
|
||||
// Commonly used to compute the number of blocks to launch in a kernel.
|
||||
int NumGroups(const int size, const int group_size) { // NOLINT
|
||||
return (size + group_size - 1) / group_size;
|
||||
}
|
||||
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
|
||||
RowMajorMatrixXf;
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
|
||||
ColMajorMatrixXf;
|
||||
|
||||
constexpr char kImageFrameTag[] = "IMAGE";
|
||||
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
|
||||
constexpr char kTensorsTag[] = "TENSORS";
|
||||
constexpr char kMatrixTag[] = "MATRIX";
|
||||
} // namespace
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Calculator for normalizing and converting an ImageFrame, GpuBuffer or Matrix
|
||||
// into a Tensor.
|
||||
//
|
||||
// This calculator is designed to be used with the TfLiteInferenceCalcualtor,
|
||||
// as a pre-processing step for calculator inputs.
|
||||
//
|
||||
// IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1],
|
||||
// specified by options (unless outputting a quantized tensor).
|
||||
//
|
||||
// Input:
|
||||
// One of the following tags:
|
||||
// IMAGE - ImageFrame (assumed to be 8-bit or 32-bit data).
|
||||
// IMAGE_GPU - GpuBuffer (assumed to be RGBA or RGB GL texture).
|
||||
// MATRIX - Matrix.
|
||||
//
|
||||
// Output:
|
||||
// One of the following tags:
|
||||
// TENSORS - Vector of Tensors of type kFloat32. The resource type used:
|
||||
// - MTLBuffer if Metal API is available
|
||||
// - SSBO if Metal is unavailable and OpenGL ES 3.1 is available
|
||||
// - Texture2D if Metal and GLES 3.1 are not available and GLES 3.0 is.
|
||||
//
|
||||
// Example use:
|
||||
// node {
|
||||
// calculator: "TensorConverterCalculator"
|
||||
// input_stream: "IMAGE:input_image"
|
||||
// output_stream: "TENSORS:image_tensor"
|
||||
// options: {
|
||||
// [mediapipe.TensorConverterCalculatorOptions.ext] {
|
||||
// zero_center: true
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// IMPORTANT Notes:
|
||||
// GPU tensors are currently only supported on mobile platforms.
|
||||
|
||||
class TensorConverterCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
::mediapipe::Status InitGpu(CalculatorContext* cc);
|
||||
::mediapipe::Status LoadOptions(CalculatorContext* cc);
|
||||
template <class T>
|
||||
::mediapipe::Status NormalizeImage(const ImageFrame& image_frame,
|
||||
bool flip_vertically, float* tensor_ptr);
|
||||
::mediapipe::Status CopyMatrixToTensor(const Matrix& matrix,
|
||||
float* tensor_ptr);
|
||||
::mediapipe::Status ProcessCPU(CalculatorContext* cc);
|
||||
::mediapipe::Status ProcessGPU(CalculatorContext* cc);
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
MPPMetalHelper* gpu_helper_ = nullptr;
|
||||
id<MTLComputePipelineState> to_buffer_program_;
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
mediapipe::GlCalculatorHelper gpu_helper_;
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
GLuint to_buffer_program_;
|
||||
#else
|
||||
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
|
||||
GLuint to_tex2d_program_;
|
||||
GLuint framebuffer_;
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
|
||||
bool initialized_ = false;
|
||||
bool use_gpu_ = false;
|
||||
absl::optional<std::pair<float, float>> output_range_;
|
||||
bool flip_vertically_ = false;
|
||||
bool row_major_matrix_ = false;
|
||||
int max_num_channels_ = 3;
|
||||
};
|
||||
REGISTER_CALCULATOR(TensorConverterCalculator);
|
||||
|
||||
::mediapipe::Status TensorConverterCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
// Confirm only one of the input streams is present.
|
||||
RET_CHECK(static_cast<int>(cc->Inputs().HasTag(kImageFrameTag)) +
|
||||
static_cast<int>(cc->Inputs().HasTag(kGpuBufferTag)) +
|
||||
static_cast<int>(cc->Inputs().HasTag(kMatrixTag)) ==
|
||||
1);
|
||||
|
||||
if (cc->Inputs().HasTag(kImageFrameTag)) {
|
||||
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
|
||||
}
|
||||
if (cc->Inputs().HasTag(kMatrixTag)) {
|
||||
cc->Inputs().Tag(kMatrixTag).Set<Matrix>();
|
||||
}
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (cc->Inputs().HasTag(kGpuBufferTag)) {
|
||||
cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
RET_CHECK(cc->Outputs().HasTag(kTensorsTag));
|
||||
cc->Outputs().Tag(kTensorsTag).Set<std::vector<Tensor>>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorConverterCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
MP_RETURN_IF_ERROR(LoadOptions(cc));
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (cc->Inputs().HasTag(kGpuBufferTag)) {
|
||||
use_gpu_ = true;
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
|
||||
RET_CHECK(gpu_helper_);
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorConverterCalculator::Process(CalculatorContext* cc) {
|
||||
if (use_gpu_) {
|
||||
if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
// Convert to GPU tensors type.
|
||||
MP_RETURN_IF_ERROR(ProcessGPU(cc));
|
||||
} else {
|
||||
// Convert to CPU tensors or Matrix type.
|
||||
MP_RETURN_IF_ERROR(ProcessCPU(cc));
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorConverterCalculator::Close(CalculatorContext* cc) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (use_gpu_) {
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
to_buffer_program_ = nil;
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
gpu_helper_.RunInGlContext([this] {
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
glDeleteProgram(to_buffer_program_);
|
||||
#else
|
||||
glDeleteFramebuffers(1, &framebuffer_);
|
||||
glDeleteProgram(to_tex2d_program_);
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
});
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorConverterCalculator::ProcessCPU(
|
||||
CalculatorContext* cc) {
|
||||
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
|
||||
if (cc->Inputs().HasTag(kImageFrameTag)) {
|
||||
if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
const auto& image_frame =
|
||||
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels_);
|
||||
const mediapipe::ImageFormat::Format format = image_frame.Format();
|
||||
|
||||
if (!(format == mediapipe::ImageFormat::SRGBA ||
|
||||
format == mediapipe::ImageFormat::SRGB ||
|
||||
format == mediapipe::ImageFormat::GRAY8 ||
|
||||
format == mediapipe::ImageFormat::VEC32F1))
|
||||
RET_CHECK_FAIL() << "Unsupported CPU input format.";
|
||||
|
||||
output_tensors->emplace_back(
|
||||
Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels_preserved});
|
||||
auto cpu_view = output_tensors->back().GetCpuWriteView();
|
||||
|
||||
// Copy image data into tensor.
|
||||
if (image_frame.ByteDepth() == 1) {
|
||||
MP_RETURN_IF_ERROR(NormalizeImage<uint8>(image_frame, flip_vertically_,
|
||||
cpu_view.buffer<float>()));
|
||||
} else if (image_frame.ByteDepth() == 4) {
|
||||
MP_RETURN_IF_ERROR(NormalizeImage<float>(image_frame, flip_vertically_,
|
||||
cpu_view.buffer<float>()));
|
||||
} else {
|
||||
return ::mediapipe::InternalError(
|
||||
"Only byte-based (8 bit) and float (32 bit) images supported.");
|
||||
}
|
||||
} else if (cc->Inputs().HasTag(kMatrixTag)) {
|
||||
if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>();
|
||||
const int height = matrix.rows();
|
||||
const int width = matrix.cols();
|
||||
const int channels = 1;
|
||||
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels});
|
||||
MP_RETURN_IF_ERROR(CopyMatrixToTensor(
|
||||
matrix, output_tensors->back().GetCpuWriteView().buffer<float>()));
|
||||
} else {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
cc->Outputs()
|
||||
.Tag(kTensorsTag)
|
||||
.Add(output_tensors.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorConverterCalculator::ProcessGPU(
|
||||
CalculatorContext* cc) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (!initialized_) {
|
||||
MP_RETURN_IF_ERROR(InitGpu(cc));
|
||||
initialized_ = true;
|
||||
}
|
||||
const auto& input =
|
||||
cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
|
||||
int width = input.width();
|
||||
int height = input.height();
|
||||
int channels = max_num_channels_;
|
||||
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
|
||||
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels});
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
id<MTLDevice> device = gpu_helper_.mtlDevice;
|
||||
id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
|
||||
command_buffer.label = @"TensorConverterCalculatorConvert";
|
||||
id<MTLComputeCommandEncoder> compute_encoder =
|
||||
[command_buffer computeCommandEncoder];
|
||||
[compute_encoder setComputePipelineState:to_buffer_program_];
|
||||
id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
|
||||
[compute_encoder setTexture:src_texture atIndex:0];
|
||||
auto output_view =
|
||||
output_tensors->at(0).GetMtlBufferWriteView(command_buffer);
|
||||
[compute_encoder setBuffer:output_view.buffer() offset:0 atIndex:1];
|
||||
MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1);
|
||||
MTLSize threadgroups =
|
||||
MTLSizeMake(NumGroups(input.width(), kWorkgroupSize),
|
||||
NumGroups(input.height(), kWorkgroupSize), 1);
|
||||
[compute_encoder dispatchThreadgroups:threadgroups
|
||||
threadsPerThreadgroup:threads_per_group];
|
||||
[compute_encoder endEncoding];
|
||||
[command_buffer commit];
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
|
||||
[this, &output_tensors, &input]() -> ::mediapipe::Status {
|
||||
auto src = gpu_helper_.CreateSourceTexture(input);
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
// Convert GL texture into SSBO.
|
||||
glActiveTexture(GL_TEXTURE0);
|
||||
glBindTexture(GL_TEXTURE_2D, src.name());
|
||||
auto output_view = output_tensors->back().GetOpenGlBufferWriteView();
|
||||
glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, output_view.name());
|
||||
glUseProgram(to_buffer_program_);
|
||||
glDispatchCompute(NumGroups(input.width(), kWorkgroupSize),
|
||||
NumGroups(input.height(), kWorkgroupSize), 1);
|
||||
glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0);
|
||||
glBindTexture(GL_TEXTURE_2D, 0);
|
||||
#else
|
||||
// Texture2D -> Texture2D with OpenGL ES 3.0.
|
||||
glUseProgram(to_tex2d_program_);
|
||||
glDisable(GL_DEPTH_TEST);
|
||||
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
|
||||
glViewport(0, 0, src.width(), src.height());
|
||||
glActiveTexture(GL_TEXTURE0);
|
||||
auto output_view = output_tensors->back().GetOpenGlTexture2dWriteView();
|
||||
glBindTexture(GL_TEXTURE_2D, output_view.name());
|
||||
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
|
||||
GL_TEXTURE_2D, output_view.name(), 0);
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(src.target(), src.name());
|
||||
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0,
|
||||
mediapipe::kBasicSquareVertices);
|
||||
glEnableVertexAttribArray(ATTRIB_VERTEX);
|
||||
glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0,
|
||||
mediapipe::kBasicTextureVertices);
|
||||
glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
|
||||
|
||||
// draw
|
||||
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
|
||||
|
||||
// cleanup
|
||||
glActiveTexture(GL_TEXTURE0);
|
||||
glBindTexture(GL_TEXTURE_2D, 0);
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(GL_TEXTURE_2D, 0);
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
src.Release();
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
cc->Outputs()
|
||||
.Tag(kTensorsTag)
|
||||
.Add(output_tensors.release(), cc->InputTimestamp());
|
||||
#else
|
||||
RET_CHECK_FAIL() << "GPU processing is not enabled.";
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
// Get input image sizes.
|
||||
const auto& input =
|
||||
cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
|
||||
mediapipe::ImageFormat::Format format =
|
||||
mediapipe::ImageFormatForGpuBufferFormat(input.format());
|
||||
const bool include_alpha = (max_num_channels_ == 4);
|
||||
const bool single_channel = (max_num_channels_ == 1);
|
||||
if (!(format == mediapipe::ImageFormat::GRAY8 ||
|
||||
format == mediapipe::ImageFormat::SRGB ||
|
||||
format == mediapipe::ImageFormat::SRGBA))
|
||||
RET_CHECK_FAIL() << "Unsupported GPU input format.";
|
||||
if (include_alpha && (format != mediapipe::ImageFormat::SRGBA))
|
||||
RET_CHECK_FAIL() << "Num input channels is less than desired output.";
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
id<MTLDevice> device = gpu_helper_.mtlDevice;
|
||||
// Shader to convert GL Texture to Metal Buffer,
|
||||
// with normalization to either: [0,1] or [-1,1].
|
||||
const std::string shader_source = absl::Substitute(
|
||||
R"(
|
||||
#include <metal_stdlib>
|
||||
|
||||
using namespace metal;
|
||||
|
||||
kernel void convertKernel(
|
||||
texture2d<half, access::sample> in_tex [[ texture(0) ]],
|
||||
device float* out_buf [[ buffer(1) ]],
|
||||
uint2 gid [[ thread_position_in_grid ]]) {
|
||||
if (gid.x >= in_tex.get_width() || gid.y >= in_tex.get_height()) return;
|
||||
constexpr sampler texture_sampler(coord::pixel, address::clamp_to_edge);
|
||||
const float2 coord = float2(gid.x, gid.y);
|
||||
half4 pixel = in_tex.sample(texture_sampler, coord);
|
||||
$0 // normalize [-1,1]
|
||||
const int linear_index = $1 * ($2 * in_tex.get_width() + gid.x);
|
||||
out_buf[linear_index + 0] = pixel.x;
|
||||
$3 // g & b channels
|
||||
$4 // alpha channel
|
||||
}
|
||||
)",
|
||||
/*$0=*/
|
||||
output_range_.has_value()
|
||||
? absl::Substitute("pixel = pixel * half($0) + half($1);",
|
||||
(output_range_->second - output_range_->first),
|
||||
output_range_->first)
|
||||
: "",
|
||||
/*$1=*/max_num_channels_,
|
||||
/*$2=*/flip_vertically_ ? "(in_tex.get_height() - 1 - gid.y)" : "gid.y",
|
||||
/*$3=*/
|
||||
single_channel ? "" : R"(out_buf[linear_index + 1] = pixel.y;
|
||||
out_buf[linear_index + 2] = pixel.z;)",
|
||||
/*$4=*/include_alpha ? "out_buf[linear_index + 3] = pixel.w;" : "");
|
||||
|
||||
NSString* library_source =
|
||||
[NSString stringWithUTF8String:shader_source.c_str()];
|
||||
NSError* error = nil;
|
||||
id<MTLLibrary> library =
|
||||
[device newLibraryWithSource:library_source options:nullptr error:&error];
|
||||
RET_CHECK(library != nil) << "Couldn't create shader library "
|
||||
<< [[error localizedDescription] UTF8String];
|
||||
id<MTLFunction> kernel_func = nil;
|
||||
kernel_func = [library newFunctionWithName:@"convertKernel"];
|
||||
RET_CHECK(kernel_func != nil) << "Couldn't create kernel function.";
|
||||
to_buffer_program_ =
|
||||
[device newComputePipelineStateWithFunction:kernel_func error:&error];
|
||||
RET_CHECK(to_buffer_program_ != nil) << "Couldn't create pipeline state " <<
|
||||
[[error localizedDescription] UTF8String];
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, &include_alpha,
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
&input,
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
&single_channel]()
|
||||
-> ::mediapipe::Status {
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
// Shader to convert GL Texture to Shader Storage Buffer Object (SSBO),
|
||||
// with normalization to either: [0,1] or [-1,1].
|
||||
const std::string shader_source = absl::Substitute(
|
||||
R"( #version 310 es
|
||||
layout(local_size_x = $0, local_size_y = $0) in;
|
||||
layout(binding = 0) uniform sampler2D input_texture;
|
||||
layout(std430, binding = 1) buffer Output {float elements[];} output_data;
|
||||
ivec2 width_height = ivec2($1, $2);
|
||||
void main() {
|
||||
ivec2 gid = ivec2(gl_GlobalInvocationID.xy);
|
||||
if (gid.x >= width_height.x || gid.y >= width_height.y) return;
|
||||
vec4 pixel = texelFetch(input_texture, gid, 0);
|
||||
$3 // normalize [-1,1]
|
||||
int linear_index = $7 * ($4 * width_height.x + gid.x);
|
||||
output_data.elements[linear_index + 0] = pixel.x; // r channel
|
||||
$5 // g & b channels
|
||||
$6 // alpha channel
|
||||
})",
|
||||
/*$0=*/kWorkgroupSize, /*$1=*/input.width(), /*$2=*/input.height(),
|
||||
/*$3=*/
|
||||
output_range_.has_value()
|
||||
? absl::Substitute("pixel = pixel * float($0) + float($1);",
|
||||
(output_range_->second - output_range_->first),
|
||||
output_range_->first)
|
||||
: "",
|
||||
/*$4=*/flip_vertically_ ? "(width_height.y - 1 - gid.y)" : "gid.y",
|
||||
/*$5=*/
|
||||
single_channel ? ""
|
||||
: R"(output_data.elements[linear_index + 1] = pixel.y;
|
||||
output_data.elements[linear_index + 2] = pixel.z;)",
|
||||
/*$6=*/
|
||||
include_alpha ? "output_data.elements[linear_index + 3] = pixel.w;"
|
||||
: "",
|
||||
/*$7=*/max_num_channels_);
|
||||
GLuint shader = glCreateShader(GL_COMPUTE_SHADER);
|
||||
const GLchar* sources[] = {shader_source.c_str()};
|
||||
glShaderSource(shader, 1, sources, NULL);
|
||||
glCompileShader(shader);
|
||||
GLint compiled = GL_FALSE;
|
||||
glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
|
||||
RET_CHECK(compiled == GL_TRUE);
|
||||
to_buffer_program_ = glCreateProgram();
|
||||
glAttachShader(to_buffer_program_, shader);
|
||||
glDeleteShader(shader);
|
||||
glLinkProgram(to_buffer_program_);
|
||||
#else
|
||||
// OpenGL ES 3.0 fragment shader Texture2d -> Texture2d conversion.
|
||||
const std::string shader_source = absl::Substitute(
|
||||
R"(
|
||||
#if __VERSION__ < 130
|
||||
#define in varying
|
||||
#endif // __VERSION__ < 130
|
||||
|
||||
#ifdef GL_ES
|
||||
#define fragColor gl_FragColor
|
||||
precision highp float;
|
||||
#else
|
||||
#define lowp
|
||||
#define mediump
|
||||
#define highp
|
||||
#define texture2D texture
|
||||
out $0 fragColor;
|
||||
#endif // defined(GL_ES)
|
||||
|
||||
in vec2 sample_coordinate;
|
||||
uniform sampler2D frame;
|
||||
|
||||
void main() {
|
||||
$1 // flip
|
||||
vec4 pixel = texture2D(frame, sample_coordinate);
|
||||
$2 // normalize [-1,1]
|
||||
fragColor.r = pixel.r; // r channel
|
||||
$3 // g & b channels
|
||||
$4 // alpha channel
|
||||
})",
|
||||
/*$0=*/single_channel ? "vec1" : "vec4",
|
||||
/*$1=*/
|
||||
flip_vertically_ ? "sample_coordinate.y = 1.0 - sample_coordinate.y;"
|
||||
: "",
|
||||
/*$2=*/output_range_.has_value()
|
||||
? absl::Substitute("pixel = pixel * float($0) + float($1);",
|
||||
(output_range_->second - output_range_->first),
|
||||
output_range_->first)
|
||||
: "",
|
||||
/*$3=*/single_channel ? "" : R"(fragColor.g = pixel.g;
|
||||
fragColor.b = pixel.b;)",
|
||||
/*$4=*/
|
||||
include_alpha ? "fragColor.a = pixel.a;"
|
||||
: (single_channel ? "" : "fragColor.a = 1.0;"));
|
||||
|
||||
const GLint attr_location[NUM_ATTRIBUTES] = {
|
||||
ATTRIB_VERTEX,
|
||||
ATTRIB_TEXTURE_POSITION,
|
||||
};
|
||||
const GLchar* attr_name[NUM_ATTRIBUTES] = {
|
||||
"position",
|
||||
"texture_coordinate",
|
||||
};
|
||||
// shader program and params
|
||||
mediapipe::GlhCreateProgram(
|
||||
mediapipe::kBasicVertexShader, shader_source.c_str(), NUM_ATTRIBUTES,
|
||||
&attr_name[0], attr_location, &to_tex2d_program_);
|
||||
RET_CHECK(to_tex2d_program_) << "Problem initializing the program.";
|
||||
glUseProgram(to_tex2d_program_);
|
||||
glUniform1i(glGetUniformLocation(to_tex2d_program_, "frame"), 1);
|
||||
glGenFramebuffers(1, &framebuffer_);
|
||||
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorConverterCalculator::LoadOptions(
|
||||
CalculatorContext* cc) {
|
||||
// Get calculator options specified in the graph.
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::TensorConverterCalculatorOptions>();
|
||||
|
||||
// if zero_center, set output float range to match [-1, 1] as specified in
|
||||
// calculator proto.
|
||||
if (options.zero_center()) {
|
||||
output_range_.emplace(std::pair<float, float>(-1.0, 1.0));
|
||||
}
|
||||
|
||||
// Custom output_tensor_float_range values.
|
||||
// If the float range is specified in pb text, use the specified values
|
||||
// instead.
|
||||
if (options.has_output_tensor_float_range()) {
|
||||
output_range_.emplace(options.output_tensor_float_range().min(),
|
||||
options.output_tensor_float_range().max());
|
||||
CHECK_GT(output_range_->second, output_range_->first);
|
||||
}
|
||||
|
||||
// Custom div and sub values.
|
||||
if (options.use_custom_normalization()) {
|
||||
output_range_.emplace(std::pair<float, float>(
|
||||
-options.custom_sub(),
|
||||
-options.custom_sub() + 255.0 / options.custom_div()));
|
||||
}
|
||||
|
||||
// Get y-flip mode.
|
||||
flip_vertically_ = options.flip_vertically();
|
||||
|
||||
// Get row_major_matrix mode.
|
||||
row_major_matrix_ = options.row_major_matrix();
|
||||
|
||||
// Get desired way to handle input channels.
|
||||
max_num_channels_ = options.max_num_channels();
|
||||
CHECK_GE(max_num_channels_, 1);
|
||||
CHECK_LE(max_num_channels_, 4);
|
||||
CHECK_NE(max_num_channels_, 2);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
template <class T>
|
||||
::mediapipe::Status TensorConverterCalculator::NormalizeImage(
|
||||
const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) {
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels_);
|
||||
const int channels_ignored = channels - channels_preserved;
|
||||
|
||||
if (output_range_.has_value()) {
|
||||
// If the output float range is set and we are not using custom
|
||||
// normalization, normalize the pixel values from [0, 255] to the specified
|
||||
// output range.
|
||||
RET_CHECK_NE(output_range_->first, output_range_->second);
|
||||
const float scale = (output_range_->second - output_range_->first) / 255.0f;
|
||||
const float bias = output_range_->first;
|
||||
|
||||
for (int i = 0; i < height; ++i) {
|
||||
const T* image_ptr = reinterpret_cast<const T*>(
|
||||
image_frame.PixelData() +
|
||||
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
|
||||
for (int j = 0; j < width; ++j) {
|
||||
for (int c = 0; c < channels_preserved; ++c) {
|
||||
*tensor_ptr++ = *image_ptr++ * scale + bias;
|
||||
}
|
||||
image_ptr += channels_ignored;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// [0,1], scale only (bias == 0)
|
||||
// Verified that there are no precision issues with 1.0f / 255.0f expression
|
||||
const float scale = 1.0f / 255.0f;
|
||||
for (int i = 0; i < height; ++i) {
|
||||
const T* image_ptr = reinterpret_cast<const T*>(
|
||||
image_frame.PixelData() +
|
||||
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
|
||||
for (int j = 0; j < width; ++j) {
|
||||
for (int c = 0; c < channels_preserved; ++c) {
|
||||
*tensor_ptr++ = *image_ptr++ * scale;
|
||||
}
|
||||
image_ptr += channels_ignored;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorConverterCalculator::CopyMatrixToTensor(
|
||||
const Matrix& matrix, float* tensor_ptr) {
|
||||
if (row_major_matrix_) {
|
||||
auto matrix_map =
|
||||
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
} else {
|
||||
auto matrix_map =
|
||||
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,69 @@
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
// Full Example:
|
||||
//
|
||||
// node {
|
||||
// calculator: "TensorConverterCalculator"
|
||||
// input_stream: "IMAGE_IN:input_image"
|
||||
// output_stream: "TENSOR_OUT:image_tensor"
|
||||
// options {
|
||||
// [mediapipe.TensorConverterCalculatorOptions.ext] {
|
||||
// zero_center: true
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
message TensorConverterCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional TensorConverterCalculatorOptions ext = 335742637;
|
||||
}
|
||||
|
||||
// Choose normalization mode for output (not applied for Matrix inputs).
|
||||
// true = [-1,1]
|
||||
// false = [0,1]
|
||||
// Ignored if using quantization.
|
||||
optional bool zero_center = 1 [default = true];
|
||||
|
||||
// Custom settings to override the internal scaling factors `div` and `sub`.
|
||||
// Both values must be set to non-negative values. Will only take effect on
|
||||
// CPU AND when |use_custom_normalization| is set to true. When these custom
|
||||
// values take effect, the |zero_center| setting above will be overriden, and
|
||||
// the normalized_value will be calculated as:
|
||||
// normalized_value = input / custom_div - custom_sub.
|
||||
optional bool use_custom_normalization = 6 [default = false];
|
||||
optional float custom_div = 7 [default = -1.0];
|
||||
optional float custom_sub = 8 [default = -1.0];
|
||||
|
||||
// Whether the input image should be flipped vertically (along the
|
||||
// y-direction). This is useful, for example, when the input image is defined
|
||||
// with a coordinate system where the origin is at the bottom-left corner
|
||||
// (e.g., in OpenGL) whereas the ML model expects an image with a top-left
|
||||
// origin.
|
||||
optional bool flip_vertically = 2 [default = false];
|
||||
|
||||
// Controls how many channels of the input image get passed through to the
|
||||
// tensor. Valid values are 1,3,4 only. Ignored for iOS GPU.
|
||||
optional int32 max_num_channels = 3 [default = 3];
|
||||
|
||||
// The calculator expects Matrix inputs to be in column-major order. Set
|
||||
// row_major_matrix to true if the inputs are in row-major order.
|
||||
optional bool row_major_matrix = 4 [default = false];
|
||||
|
||||
// Quantization option (CPU only).
|
||||
// When true, output kUint8 tensor instead of kFloat32.
|
||||
optional bool use_quantized_tensors = 5 [default = false];
|
||||
|
||||
// Normalization option.
|
||||
// Setting normalization_range results in the values normalized to
|
||||
// the range [output_tensor_float_range.min, output_tensor_float_range.max].
|
||||
optional TensorFloatRange output_tensor_float_range = 9;
|
||||
|
||||
message TensorFloatRange {
|
||||
optional float min = 1;
|
||||
optional float max = 2;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
constexpr char kTransposeOptionsString[] =
|
||||
"[mediapipe.TensorConverterCalculatorOptions.ext]: {"
|
||||
"row_major_matrix: True}";
|
||||
|
||||
} // namespace
|
||||
|
||||
using RandomEngine = std::mt19937_64;
|
||||
using testing::Eq;
|
||||
const uint32 kSeed = 1234;
|
||||
const int kNumSizes = 8;
|
||||
const int sizes[kNumSizes][2] = {{1, 1}, {12, 1}, {1, 9}, {2, 2},
|
||||
{5, 3}, {7, 13}, {16, 32}, {101, 2}};
|
||||
|
||||
class TensorConverterCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
// Adds a packet with a matrix filled with random values in [0,1].
|
||||
void AddRandomMatrix(int num_rows, int num_columns, uint32 seed,
|
||||
bool row_major_matrix = false) {
|
||||
RandomEngine random(kSeed);
|
||||
std::uniform_real_distribution<> uniform_dist(0, 1.0);
|
||||
auto matrix = ::absl::make_unique<Matrix>();
|
||||
matrix->resize(num_rows, num_columns);
|
||||
if (row_major_matrix) {
|
||||
for (int y = 0; y < num_rows; ++y) {
|
||||
for (int x = 0; x < num_columns; ++x) {
|
||||
float value = uniform_dist(random);
|
||||
(*matrix)(y, x) = value;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int x = 0; x < num_columns; ++x) {
|
||||
for (int y = 0; y < num_rows; ++y) {
|
||||
float value = uniform_dist(random);
|
||||
(*matrix)(y, x) = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
MP_ASSERT_OK(graph_->AddPacketToInputStream(
|
||||
"matrix", Adopt(matrix.release()).At(Timestamp(0))));
|
||||
}
|
||||
|
||||
std::unique_ptr<CalculatorGraph> graph_;
|
||||
};
|
||||
|
||||
TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
|
||||
for (int size_index = 0; size_index < kNumSizes; ++size_index) {
|
||||
const int num_rows = sizes[size_index][0];
|
||||
const int num_columns = sizes[size_index][1];
|
||||
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
input_stream: "MATRIX:matrix"
|
||||
output_stream: "TENSORS:tensor"
|
||||
options {
|
||||
[mediapipe.TensorConverterCalculatorOptions.ext] {
|
||||
row_major_matrix: false
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
graph_ = absl::make_unique<CalculatorGraph>();
|
||||
MP_ASSERT_OK(graph_->Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph_->StartRun({}));
|
||||
|
||||
// Push the tensor into the graph.
|
||||
AddRandomMatrix(num_rows, num_columns, kSeed, /*row_major_matrix=*/false);
|
||||
|
||||
// Wait until the calculator done processing.
|
||||
MP_ASSERT_OK(graph_->WaitUntilIdle());
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
|
||||
// Get and process results.
|
||||
const std::vector<Tensor>& tensor_vec =
|
||||
output_packets[0].Get<std::vector<Tensor>>();
|
||||
EXPECT_EQ(1, tensor_vec.size());
|
||||
|
||||
const Tensor* tensor = &tensor_vec[0];
|
||||
EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type());
|
||||
|
||||
// Verify that the data is correct.
|
||||
RandomEngine random(kSeed);
|
||||
std::uniform_real_distribution<> uniform_dist(0, 1.0);
|
||||
auto view = tensor->GetCpuReadView();
|
||||
auto tensor_buffer = view.buffer<float>();
|
||||
for (int i = 0; i < num_rows * num_columns; ++i) {
|
||||
const float expected = uniform_dist(random);
|
||||
EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i;
|
||||
}
|
||||
|
||||
// Fully close graph at end, otherwise calculator+tensors are destroyed
|
||||
// after calling WaitUntilDone().
|
||||
MP_ASSERT_OK(graph_->CloseInputStream("matrix"));
|
||||
MP_ASSERT_OK(graph_->WaitUntilDone());
|
||||
|
||||
graph_.reset();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
|
||||
for (int size_index = 0; size_index < kNumSizes; ++size_index) {
|
||||
const int num_rows = sizes[size_index][0];
|
||||
const int num_columns = sizes[size_index][1];
|
||||
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
input_stream: "MATRIX:matrix"
|
||||
output_stream: "TENSORS:tensor"
|
||||
options {
|
||||
[mediapipe.TensorConverterCalculatorOptions.ext] {
|
||||
row_major_matrix: true
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
graph_ = absl::make_unique<CalculatorGraph>();
|
||||
MP_ASSERT_OK(graph_->Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph_->StartRun({}));
|
||||
|
||||
// Push the tensor into the graph.
|
||||
AddRandomMatrix(num_rows, num_columns, kSeed, /*row_major_matrix=*/true);
|
||||
|
||||
// Wait until the calculator done processing.
|
||||
MP_ASSERT_OK(graph_->WaitUntilIdle());
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
|
||||
// Get and process results.
|
||||
const std::vector<Tensor>& tensor_vec =
|
||||
output_packets[0].Get<std::vector<Tensor>>();
|
||||
EXPECT_EQ(1, tensor_vec.size());
|
||||
|
||||
const Tensor* tensor = &tensor_vec[0];
|
||||
EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type());
|
||||
|
||||
// Verify that the data is correct.
|
||||
RandomEngine random(kSeed);
|
||||
std::uniform_real_distribution<> uniform_dist(0, 1.0);
|
||||
auto view = tensor->GetCpuReadView();
|
||||
auto tensor_buffer = view.buffer<float>();
|
||||
for (int i = 0; i < num_rows * num_columns; ++i) {
|
||||
const float expected = uniform_dist(random);
|
||||
EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i;
|
||||
}
|
||||
|
||||
// Fully close graph at end, otherwise calculator+tensors are destroyed
|
||||
// after calling WaitUntilDone().
|
||||
MP_ASSERT_OK(graph_->CloseInputStream("matrix"));
|
||||
MP_ASSERT_OK(graph_->WaitUntilDone());
|
||||
|
||||
graph_.reset();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
|
||||
CalculatorGraph graph;
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: "input_image"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
input_stream: "IMAGE:input_image"
|
||||
output_stream: "TENSORS:tensor"
|
||||
options {
|
||||
[mediapipe.TensorConverterCalculatorOptions.ext] {
|
||||
row_major_matrix: true
|
||||
use_custom_normalization: true
|
||||
custom_div: 2.0
|
||||
custom_sub: 33.0
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
auto input_image = absl::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
|
||||
cv::Mat mat = ::mediapipe::formats::MatView(input_image.get());
|
||||
mat.at<uint8>(0, 0) = 200;
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
|
||||
|
||||
// Wait until the calculator done processing.
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
// Get and process results.
|
||||
const std::vector<Tensor>& tensor_vec =
|
||||
output_packets[0].Get<std::vector<Tensor>>();
|
||||
EXPECT_EQ(1, tensor_vec.size());
|
||||
|
||||
const Tensor* tensor = &tensor_vec[0];
|
||||
EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type());
|
||||
auto view = tensor->GetCpuReadView();
|
||||
EXPECT_FLOAT_EQ(67.0f, *view.buffer<float>());
|
||||
|
||||
// Fully close graph at end, otherwise calculator+tensors are destroyed
|
||||
// after calling WaitUntilDone().
|
||||
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
|
||||
std::vector<std::pair<float, float>> range_values = {
|
||||
std::make_pair(0.0, 1.0), std::make_pair(-1.0, 1.0),
|
||||
std::make_pair(-0.5, 0.5)};
|
||||
for (std::pair<float, float> range : range_values) {
|
||||
CalculatorGraph graph;
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
absl::Substitute(R"(
|
||||
input_stream: "input_image"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
input_stream: "IMAGE:input_image"
|
||||
output_stream: "TENSORS:tensor"
|
||||
options {
|
||||
[mediapipe.TensorConverterCalculatorOptions.ext] {
|
||||
output_tensor_float_range {
|
||||
min: $0
|
||||
max: $1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)",
|
||||
/*$0=*/range.first,
|
||||
/*$1=*/range.second));
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
auto input_image = absl::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
|
||||
cv::Mat mat = ::mediapipe::formats::MatView(input_image.get());
|
||||
mat.at<uint8>(0, 0) = 200;
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
|
||||
|
||||
// Wait until the calculator finishes processing.
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
EXPECT_THAT(output_packets.size(), Eq(1));
|
||||
|
||||
// Get and process results.
|
||||
const std::vector<Tensor>& tensor_vec =
|
||||
output_packets[0].Get<std::vector<Tensor>>();
|
||||
EXPECT_THAT(tensor_vec.size(), Eq(1));
|
||||
|
||||
const Tensor* tensor = &tensor_vec[0];
|
||||
|
||||
// Calculate the expected normalized value:
|
||||
float normalized_value =
|
||||
range.first + (200 * (range.second - range.first)) / 255.0;
|
||||
|
||||
EXPECT_THAT(tensor->element_type(), Eq(Tensor::ElementType::kFloat32));
|
||||
auto view = tensor->GetCpuReadView();
|
||||
float dataf = *view.buffer<float>();
|
||||
EXPECT_THAT(
|
||||
normalized_value,
|
||||
testing::FloatNear(dataf, 2.0f * std::abs(dataf) *
|
||||
std::numeric_limits<float>::epsilon()));
|
||||
|
||||
// Fully close graph at end, otherwise calculator+tensors are destroyed
|
||||
// after calling WaitUntilDone().
|
||||
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,197 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "absl/types/span.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_classification_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
#if defined(MEDIAPIPE_MOBILE)
|
||||
#include "mediapipe/util/android/file/base/file.h"
|
||||
#include "mediapipe/util/android/file/base/helpers.h"
|
||||
#else
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#endif
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Convert result tensors from classification models into MediaPipe
|
||||
// classifications.
|
||||
//
|
||||
// Input:
|
||||
// TENSORS - Vector of Tensors of type kFloat32 containing one
|
||||
// tensor, the size of which must be (1, * num_classes).
|
||||
// Output:
|
||||
// CLASSIFICATIONS - Result MediaPipe ClassificationList. The score and index
|
||||
// fields of each classification are set, while the label
|
||||
// field is only set if label_map_path is provided.
|
||||
//
|
||||
// Usage example:
|
||||
// node {
|
||||
// calculator: "TensorsToClassificationCalculator"
|
||||
// input_stream: "TENSORS:tensors"
|
||||
// output_stream: "CLASSIFICATIONS:classifications"
|
||||
// options: {
|
||||
// [mediapipe.TensorsToClassificationCalculatorOptions.ext] {
|
||||
// num_classes: 1024
|
||||
// min_score_threshold: 0.1
|
||||
// label_map_path: "labelmap.txt"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class TensorsToClassificationCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
::mediapipe::TensorsToClassificationCalculatorOptions options_;
|
||||
int top_k_ = 0;
|
||||
std::unordered_map<int, std::string> label_map_;
|
||||
bool label_map_loaded_ = false;
|
||||
};
|
||||
REGISTER_CALCULATOR(TensorsToClassificationCalculator);
|
||||
|
||||
::mediapipe::Status TensorsToClassificationCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(!cc->Inputs().GetTags().empty());
|
||||
RET_CHECK(!cc->Outputs().GetTags().empty());
|
||||
|
||||
if (cc->Inputs().HasTag("TENSORS")) {
|
||||
cc->Inputs().Tag("TENSORS").Set<std::vector<Tensor>>();
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag("CLASSIFICATIONS")) {
|
||||
cc->Outputs().Tag("CLASSIFICATIONS").Set<ClassificationList>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorsToClassificationCalculator::Open(
|
||||
CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
options_ =
|
||||
cc->Options<::mediapipe::TensorsToClassificationCalculatorOptions>();
|
||||
|
||||
top_k_ = options_.top_k();
|
||||
if (options_.has_label_map_path()) {
|
||||
std::string string_path;
|
||||
ASSIGN_OR_RETURN(string_path,
|
||||
PathToResourceAsFile(options_.label_map_path()));
|
||||
std::string label_map_string;
|
||||
MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
|
||||
|
||||
std::istringstream stream(label_map_string);
|
||||
std::string line;
|
||||
int i = 0;
|
||||
while (std::getline(stream, line)) {
|
||||
label_map_[i++] = line;
|
||||
}
|
||||
label_map_loaded_ = true;
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorsToClassificationCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
const auto& input_tensors =
|
||||
cc->Inputs().Tag("TENSORS").Get<std::vector<Tensor>>();
|
||||
|
||||
RET_CHECK_EQ(input_tensors.size(), 1);
|
||||
|
||||
int num_classes = input_tensors[0].shape().num_elements();
|
||||
|
||||
if (options_.binary_classification()) {
|
||||
RET_CHECK_EQ(num_classes, 1);
|
||||
// Number of classes for binary classification.
|
||||
num_classes = 2;
|
||||
}
|
||||
if (label_map_loaded_) {
|
||||
RET_CHECK_EQ(num_classes, label_map_.size());
|
||||
}
|
||||
auto view = input_tensors[0].GetCpuReadView();
|
||||
auto raw_scores = view.buffer<float>();
|
||||
|
||||
auto classification_list = absl::make_unique<ClassificationList>();
|
||||
if (options_.binary_classification()) {
|
||||
Classification* class_first = classification_list->add_classification();
|
||||
Classification* class_second = classification_list->add_classification();
|
||||
class_first->set_index(0);
|
||||
class_second->set_index(1);
|
||||
class_first->set_score(raw_scores[0]);
|
||||
class_second->set_score(1. - raw_scores[0]);
|
||||
|
||||
if (label_map_loaded_) {
|
||||
class_first->set_label(label_map_[0]);
|
||||
class_second->set_label(label_map_[1]);
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < num_classes; ++i) {
|
||||
if (options_.has_min_score_threshold() &&
|
||||
raw_scores[i] < options_.min_score_threshold()) {
|
||||
continue;
|
||||
}
|
||||
Classification* classification =
|
||||
classification_list->add_classification();
|
||||
classification->set_index(i);
|
||||
classification->set_score(raw_scores[i]);
|
||||
|
||||
if (label_map_loaded_) {
|
||||
classification->set_label(label_map_[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Note that partial_sort will raise error when top_k_ >
|
||||
// classification_list->classification_size().
|
||||
CHECK_GE(classification_list->classification_size(), top_k_);
|
||||
auto raw_classification_list = classification_list->mutable_classification();
|
||||
if (top_k_ > 0 && classification_list->classification_size() >= top_k_) {
|
||||
std::partial_sort(raw_classification_list->begin(),
|
||||
raw_classification_list->begin() + top_k_,
|
||||
raw_classification_list->end(),
|
||||
[](const Classification a, const Classification b) {
|
||||
return a.score() > b.score();
|
||||
});
|
||||
|
||||
// Resizes the underlying list to have only top_k_ classifications.
|
||||
raw_classification_list->DeleteSubrange(
|
||||
top_k_, raw_classification_list->size() - top_k_);
|
||||
}
|
||||
cc->Outputs()
|
||||
.Tag("CLASSIFICATIONS")
|
||||
.Add(classification_list.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorsToClassificationCalculator::Close(
|
||||
CalculatorContext* cc) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The option proto for the TensorsToClassificationCalculator.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message TensorsToClassificationCalculatorOptions {
|
||||
extend .mediapipe.CalculatorOptions {
|
||||
optional TensorsToClassificationCalculatorOptions ext = 335742638;
|
||||
}
|
||||
|
||||
// Score threshold for perserving the class.
|
||||
optional float min_score_threshold = 1;
|
||||
// Number of highest scoring labels to output. If top_k is not positive then
|
||||
// all labels are used.
|
||||
optional int32 top_k = 2;
|
||||
// Path to a label map file for getting the actual name of class ids.
|
||||
optional string label_map_path = 3;
|
||||
// Whether the input is a single float for binary classification.
|
||||
// When true, only a single float is expected in the input tensor and the
|
||||
// label map, if provided, is expected to have exactly two labels.
|
||||
// The single score(float) represent the probability of first label, and
|
||||
// 1 - score is the probabilility of the second label.
|
||||
optional bool binary_classification = 4;
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_classification_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
using ::mediapipe::ParseTextProtoOrDie;
|
||||
using Node = ::mediapipe::CalculatorGraphConfig::Node;
|
||||
|
||||
class TensorsToClassificationCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void BuildGraph(mediapipe::CalculatorRunner* runner,
|
||||
const std::vector<float>& scores) {
|
||||
auto tensors = absl::make_unique<std::vector<Tensor>>();
|
||||
tensors->emplace_back(
|
||||
Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, 1, static_cast<int>(scores.size()), 1});
|
||||
auto view = tensors->back().GetCpuWriteView();
|
||||
float* tensor_buffer = view.buffer<float>();
|
||||
ASSERT_NE(tensor_buffer, nullptr);
|
||||
for (int i = 0; i < scores.size(); ++i) {
|
||||
tensor_buffer[i] = scores[i];
|
||||
}
|
||||
|
||||
int64 stream_timestamp = 0;
|
||||
auto& input_stream_packets =
|
||||
runner->MutableInputs()->Tag("TENSORS").packets;
|
||||
|
||||
input_stream_packets.push_back(
|
||||
mediapipe::Adopt(tensors.release())
|
||||
.At(mediapipe::Timestamp(stream_timestamp++)));
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {}
|
||||
}
|
||||
)"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
|
||||
|
||||
EXPECT_EQ(1, output_packets_.size());
|
||||
|
||||
const auto& classification_list =
|
||||
output_packets_[0].Get<ClassificationList>();
|
||||
EXPECT_EQ(3, classification_list.classification_size());
|
||||
|
||||
// Verify that the label_id and score fields are set correctly.
|
||||
for (int i = 0; i < classification_list.classification_size(); ++i) {
|
||||
EXPECT_EQ(i, classification_list.classification(i).index());
|
||||
EXPECT_EQ(i * 0.5, classification_list.classification(i).score());
|
||||
ASSERT_FALSE(classification_list.classification(i).has_label());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {
|
||||
label_map_path: "mediapipe/calculators/tensor/testdata/labelmap.txt"
|
||||
}
|
||||
}
|
||||
)"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
|
||||
|
||||
EXPECT_EQ(1, output_packets_.size());
|
||||
|
||||
const auto& classification_list =
|
||||
output_packets_[0].Get<ClassificationList>();
|
||||
EXPECT_EQ(3, classification_list.classification_size());
|
||||
|
||||
// Verify that the label field is set.
|
||||
for (int i = 0; i < classification_list.classification_size(); ++i) {
|
||||
EXPECT_EQ(i, classification_list.classification(i).index());
|
||||
EXPECT_EQ(i * 0.5, classification_list.classification(i).score());
|
||||
ASSERT_TRUE(classification_list.classification(i).has_label());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest,
|
||||
CorrectOutputWithLabelMinScoreThreshold) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {
|
||||
min_score_threshold: 0.6
|
||||
}
|
||||
}
|
||||
)"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
|
||||
|
||||
EXPECT_EQ(1, output_packets_.size());
|
||||
|
||||
const auto& classification_list =
|
||||
output_packets_[0].Get<ClassificationList>();
|
||||
|
||||
// Verify that the low score labels are filtered out.
|
||||
EXPECT_EQ(1, classification_list.classification_size());
|
||||
EXPECT_EQ(1, classification_list.classification(0).score());
|
||||
}
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] { top_k: 2 }
|
||||
}
|
||||
)"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
|
||||
|
||||
EXPECT_EQ(1, output_packets_.size());
|
||||
|
||||
const auto& classification_list =
|
||||
output_packets_[0].Get<ClassificationList>();
|
||||
|
||||
// Verify that the only top2 labels are left.
|
||||
EXPECT_EQ(2, classification_list.classification_size());
|
||||
for (int i = 0; i < classification_list.classification_size(); ++i) {
|
||||
EXPECT_EQ((classification_list.classification_size() - i) * 0.5,
|
||||
classification_list.classification(i).score());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,74 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The option proto for the TensorsToDetectionsCalculator.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message TensorsToDetectionsCalculatorOptions {
|
||||
extend .mediapipe.CalculatorOptions {
|
||||
optional TensorsToDetectionsCalculatorOptions ext = 335742639;
|
||||
}
|
||||
|
||||
// [Required] The number of output classes predicted by the detection model.
|
||||
optional int32 num_classes = 1;
|
||||
// [Required] The number of output boxes predicted by the detection model.
|
||||
optional int32 num_boxes = 2;
|
||||
// [Required] The number of output values per boxes predicted by the detection
|
||||
// model. The values contain bounding boxes, keypoints, etc.
|
||||
optional int32 num_coords = 3;
|
||||
|
||||
// The offset of keypoint coordinates in the location tensor.
|
||||
optional int32 keypoint_coord_offset = 9;
|
||||
// The number of predicted keypoints.
|
||||
optional int32 num_keypoints = 10 [default = 0];
|
||||
// The dimension of each keypoint, e.g. number of values predicted for each
|
||||
// keypoint.
|
||||
optional int32 num_values_per_keypoint = 11 [default = 2];
|
||||
// The offset of box coordinates in the location tensor.
|
||||
optional int32 box_coord_offset = 12 [default = 0];
|
||||
|
||||
// Parameters for decoding SSD detection model.
|
||||
optional float x_scale = 4 [default = 0.0];
|
||||
optional float y_scale = 5 [default = 0.0];
|
||||
optional float w_scale = 6 [default = 0.0];
|
||||
optional float h_scale = 7 [default = 0.0];
|
||||
|
||||
optional bool apply_exponential_on_box_size = 13 [default = false];
|
||||
|
||||
// Whether to reverse the order of predicted x, y from output.
|
||||
// If false, the order is [y_center, x_center, h, w], if true the order is
|
||||
// [x_center, y_center, w, h].
|
||||
optional bool reverse_output_order = 14 [default = false];
|
||||
// The ids of classes that should be ignored during decoding the score for
|
||||
// each predicted box.
|
||||
repeated int32 ignore_classes = 8;
|
||||
|
||||
optional bool sigmoid_score = 15 [default = false];
|
||||
optional float score_clipping_thresh = 16;
|
||||
|
||||
// Whether the detection coordinates from the input tensors should be flipped
|
||||
// vertically (along the y-direction). This is useful, for example, when the
|
||||
// input tensors represent detections defined with a coordinate system where
|
||||
// the origin is at the top-left corner, whereas the desired detection
|
||||
// representation has a bottom-left origin (e.g., in OpenGL).
|
||||
optional bool flip_vertically = 18 [default = false];
|
||||
|
||||
// Score threshold for perserving decoded detections.
|
||||
optional float min_score_thresh = 19;
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// A calculator for converting Tensors to to a float or a float vector.
|
||||
//
|
||||
// Input:
|
||||
// TENSORS - Vector of Tensors of type kFloat32. Only the first
|
||||
// tensor will be used.
|
||||
// Output:
|
||||
// FLOAT(optional) - Converted single float number.
|
||||
// FLOATS(optional) - Converted float vector.
|
||||
//
|
||||
// Notes: To output FLOAT stream, the input tensor must have size 1, e.g.
|
||||
// only 1 float number in the tensor.
|
||||
//
|
||||
// Usage example:
|
||||
// node {
|
||||
// calculator: "TensorsToFloatsCalculator"
|
||||
// input_stream: "TENSORS:tensors"
|
||||
// output_stream: "FLOATS:floats"
|
||||
// }
|
||||
class TensorsToFloatsCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(TensorsToFloatsCalculator);
|
||||
|
||||
::mediapipe::Status TensorsToFloatsCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag("TENSORS"));
|
||||
RET_CHECK(cc->Outputs().HasTag("FLOATS") || cc->Outputs().HasTag("FLOAT"));
|
||||
|
||||
cc->Inputs().Tag("TENSORS").Set<std::vector<Tensor>>();
|
||||
if (cc->Outputs().HasTag("FLOATS")) {
|
||||
cc->Outputs().Tag("FLOATS").Set<std::vector<float>>();
|
||||
}
|
||||
if (cc->Outputs().HasTag("FLOAT")) {
|
||||
cc->Outputs().Tag("FLOAT").Set<float>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorsToFloatsCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorsToFloatsCalculator::Process(CalculatorContext* cc) {
|
||||
RET_CHECK(!cc->Inputs().Tag("TENSORS").IsEmpty());
|
||||
|
||||
const auto& input_tensors =
|
||||
cc->Inputs().Tag("TENSORS").Get<std::vector<Tensor>>();
|
||||
// TODO: Add option to specify which tensor to take from.
|
||||
auto view = input_tensors[0].GetCpuReadView();
|
||||
auto raw_floats = view.buffer<float>();
|
||||
int num_values = input_tensors[0].shape().num_elements();
|
||||
|
||||
if (cc->Outputs().HasTag("FLOAT")) {
|
||||
// TODO: Could add an index in the option to specifiy returning one
|
||||
// value of a float array.
|
||||
RET_CHECK_EQ(num_values, 1);
|
||||
cc->Outputs().Tag("FLOAT").AddPacket(
|
||||
MakePacket<float>(raw_floats[0]).At(cc->InputTimestamp()));
|
||||
}
|
||||
if (cc->Outputs().HasTag("FLOATS")) {
|
||||
auto output_floats = absl::make_unique<std::vector<float>>(
|
||||
raw_floats, raw_floats + num_values);
|
||||
cc->Outputs().Tag("FLOATS").Add(output_floats.release(),
|
||||
cc->InputTimestamp());
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,250 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_landmarks_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// A calculator for converting Tensors from regression models into landmarks.
|
||||
// Note that if the landmarks in the tensor has more than 5 dimensions, only the
|
||||
// first 5 dimensions will be converted to [x,y,z, visibility, presence].
|
||||
//
|
||||
// Input:
|
||||
// TENSORS - Vector of Tensors of type kFloat32. Only the first tensor will be
|
||||
// used. The size of the values must be (num_dimension x num_landmarks).
|
||||
//
|
||||
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
|
||||
// not. Overrides corresponding side packet and/or field in the calculator
|
||||
// options.
|
||||
//
|
||||
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
|
||||
// Overrides corresponding side packet and/or field in the calculator options.
|
||||
//
|
||||
// Input side packet:
|
||||
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
|
||||
// not. Overrides the corresponding field in the calculator options.
|
||||
//
|
||||
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
|
||||
// Overrides the corresponding field in the calculator options.
|
||||
//
|
||||
// Output:
|
||||
// LANDMARKS(optional) - Result MediaPipe landmarks.
|
||||
// NORM_LANDMARKS(optional) - Result MediaPipe normalized landmarks.
|
||||
//
|
||||
// Notes:
|
||||
// To output normalized landmarks, user must provide the original input image
|
||||
// size to the model using calculator option input_image_width and
|
||||
// input_image_height.
|
||||
// Usage example:
|
||||
// node {
|
||||
// calculator: "TensorsToLandmarksCalculator"
|
||||
// input_stream: "TENSORS:landmark_tensors"
|
||||
// output_stream: "LANDMARKS:landmarks"
|
||||
// output_stream: "NORM_LANDMARKS:landmarks"
|
||||
// options: {
|
||||
// [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
|
||||
// num_landmarks: 21
|
||||
//
|
||||
// input_image_width: 256
|
||||
// input_image_height: 256
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class TensorsToLandmarksCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
::mediapipe::Status LoadOptions(CalculatorContext* cc);
|
||||
int num_landmarks_ = 0;
|
||||
bool flip_vertically_ = false;
|
||||
bool flip_horizontally_ = false;
|
||||
|
||||
::mediapipe::TensorsToLandmarksCalculatorOptions options_;
|
||||
};
|
||||
REGISTER_CALCULATOR(TensorsToLandmarksCalculator);
|
||||
|
||||
::mediapipe::Status TensorsToLandmarksCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(!cc->Inputs().GetTags().empty());
|
||||
RET_CHECK(!cc->Outputs().GetTags().empty());
|
||||
|
||||
if (cc->Inputs().HasTag("TENSORS")) {
|
||||
cc->Inputs().Tag("TENSORS").Set<std::vector<Tensor>>();
|
||||
}
|
||||
|
||||
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
|
||||
cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
|
||||
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
|
||||
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
|
||||
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag("LANDMARKS")) {
|
||||
cc->Outputs().Tag("LANDMARKS").Set<LandmarkList>();
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag("NORM_LANDMARKS")) {
|
||||
cc->Outputs().Tag("NORM_LANDMARKS").Set<NormalizedLandmarkList>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
MP_RETURN_IF_ERROR(LoadOptions(cc));
|
||||
|
||||
if (cc->Outputs().HasTag("NORM_LANDMARKS")) {
|
||||
RET_CHECK(options_.has_input_image_height() &&
|
||||
options_.has_input_image_width())
|
||||
<< "Must provide input with/height for getting normalized landmarks.";
|
||||
}
|
||||
if (cc->Outputs().HasTag("LANDMARKS") &&
|
||||
(options_.flip_vertically() || options_.flip_horizontally() ||
|
||||
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") ||
|
||||
cc->InputSidePackets().HasTag("FLIP_VERTICALLY"))) {
|
||||
RET_CHECK(options_.has_input_image_height() &&
|
||||
options_.has_input_image_width())
|
||||
<< "Must provide input with/height for using flip_vertically option "
|
||||
"when outputing landmarks in absolute coordinates.";
|
||||
}
|
||||
|
||||
flip_horizontally_ =
|
||||
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")
|
||||
? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>()
|
||||
: options_.flip_horizontally();
|
||||
|
||||
flip_vertically_ =
|
||||
cc->InputSidePackets().HasTag("FLIP_VERTICALLY")
|
||||
? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>()
|
||||
: options_.flip_vertically();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorsToLandmarksCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
// Override values if specified so.
|
||||
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
|
||||
!cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
|
||||
flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
|
||||
}
|
||||
if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
|
||||
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
|
||||
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
|
||||
}
|
||||
|
||||
if (cc->Inputs().Tag("TENSORS").IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
const auto& input_tensors =
|
||||
cc->Inputs().Tag("TENSORS").Get<std::vector<Tensor>>();
|
||||
|
||||
int num_values = input_tensors[0].shape().num_elements();
|
||||
const int num_dimensions = num_values / num_landmarks_;
|
||||
CHECK_GT(num_dimensions, 0);
|
||||
|
||||
auto view = input_tensors[0].GetCpuReadView();
|
||||
auto raw_landmarks = view.buffer<float>();
|
||||
|
||||
LandmarkList output_landmarks;
|
||||
|
||||
for (int ld = 0; ld < num_landmarks_; ++ld) {
|
||||
const int offset = ld * num_dimensions;
|
||||
Landmark* landmark = output_landmarks.add_landmark();
|
||||
|
||||
if (flip_horizontally_) {
|
||||
landmark->set_x(options_.input_image_width() - raw_landmarks[offset]);
|
||||
} else {
|
||||
landmark->set_x(raw_landmarks[offset]);
|
||||
}
|
||||
if (num_dimensions > 1) {
|
||||
if (flip_vertically_) {
|
||||
landmark->set_y(options_.input_image_height() -
|
||||
raw_landmarks[offset + 1]);
|
||||
} else {
|
||||
landmark->set_y(raw_landmarks[offset + 1]);
|
||||
}
|
||||
}
|
||||
if (num_dimensions > 2) {
|
||||
landmark->set_z(raw_landmarks[offset + 2]);
|
||||
}
|
||||
if (num_dimensions > 3) {
|
||||
landmark->set_visibility(raw_landmarks[offset + 3]);
|
||||
}
|
||||
if (num_dimensions > 4) {
|
||||
landmark->set_presence(raw_landmarks[offset + 4]);
|
||||
}
|
||||
}
|
||||
|
||||
// Output normalized landmarks if required.
|
||||
if (cc->Outputs().HasTag("NORM_LANDMARKS")) {
|
||||
NormalizedLandmarkList output_norm_landmarks;
|
||||
for (int i = 0; i < output_landmarks.landmark_size(); ++i) {
|
||||
const Landmark& landmark = output_landmarks.landmark(i);
|
||||
NormalizedLandmark* norm_landmark = output_norm_landmarks.add_landmark();
|
||||
norm_landmark->set_x(landmark.x() / options_.input_image_width());
|
||||
norm_landmark->set_y(landmark.y() / options_.input_image_height());
|
||||
// Scale Z coordinate as X + allow additional uniform normalization.
|
||||
norm_landmark->set_z(landmark.z() / options_.input_image_width() /
|
||||
options_.normalize_z());
|
||||
norm_landmark->set_visibility(landmark.visibility());
|
||||
norm_landmark->set_presence(landmark.presence());
|
||||
}
|
||||
cc->Outputs()
|
||||
.Tag("NORM_LANDMARKS")
|
||||
.AddPacket(MakePacket<NormalizedLandmarkList>(output_norm_landmarks)
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
// Output absolute landmarks.
|
||||
if (cc->Outputs().HasTag("LANDMARKS")) {
|
||||
cc->Outputs()
|
||||
.Tag("LANDMARKS")
|
||||
.AddPacket(MakePacket<LandmarkList>(output_landmarks)
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TensorsToLandmarksCalculator::LoadOptions(
|
||||
CalculatorContext* cc) {
|
||||
// Get calculator options specified in the graph.
|
||||
options_ = cc->Options<::mediapipe::TensorsToLandmarksCalculatorOptions>();
|
||||
RET_CHECK(options_.has_num_landmarks());
|
||||
num_landmarks_ = options_.num_landmarks();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
} // namespace mediapipe
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user