Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 MediaPipe Teamandjqtang
|
710fb3de58 | ||
|
MediaPipe Teamandchuoling
|
b899d17f18 | ||
|
MediaPipe Teamandchuoling
|
50c92c6623 |
@@ -45,7 +45,7 @@ Hair Segmentation
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
@@ -79,6 +79,13 @@ run code search using
|
||||
|
||||
## Publications
|
||||
|
||||
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
||||
in Google Developers Blog
|
||||
* [Prosthesis control via Mirru App using MediaPipe hand tracking](https://developers.googleblog.com/2021/05/control-your-mirru-prosthesis-with-mediapipe-hand-tracking.html)
|
||||
in Google Developers Blog
|
||||
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
||||
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
||||
in Google Developers Blog
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
in Google AI Blog
|
||||
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
||||
|
||||
@@ -53,19 +53,12 @@ rules_foreign_cc_dependencies()
|
||||
all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])"""
|
||||
|
||||
# GoogleTest/GoogleMock framework. Used by most unit-tests.
|
||||
# Last updated 2020-06-30.
|
||||
# Last updated 2021-07-02.
|
||||
http_archive(
|
||||
name = "com_google_googletest",
|
||||
urls = ["https://github.com/google/googletest/archive/aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e.zip"],
|
||||
patches = [
|
||||
# fix for https://github.com/google/googletest/issues/2817
|
||||
"@//third_party:com_google_googletest_9d580ea80592189e6d44fa35bcf9cdea8bf620d6.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "googletest-aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e",
|
||||
sha256 = "04a1751f94244307cebe695a69cc945f9387a80b0ef1af21394a490697c5c895",
|
||||
urls = ["https://github.com/google/googletest/archive/4ec4cd23f486bf70efcc5d2caa40f24368f752e3.zip"],
|
||||
strip_prefix = "googletest-4ec4cd23f486bf70efcc5d2caa40f24368f752e3",
|
||||
sha256 = "de682ea824bfffba05b4e33b67431c247397d6175962534305136aa06f92e049",
|
||||
)
|
||||
|
||||
# Google Benchmark library.
|
||||
@@ -338,7 +331,9 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
|
||||
maven_install(
|
||||
artifacts = [
|
||||
"androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
"androidx.lifecycle:lifecycle-common:2.2.0",
|
||||
"androidx.lifecycle:lifecycle-common:2.3.1",
|
||||
"androidx.activity:activity:1.2.2",
|
||||
"androidx.fragment:fragment:1.3.4",
|
||||
"androidx.annotation:annotation:aar:1.1.0",
|
||||
"androidx.appcompat:appcompat:aar:1.1.0-rc01",
|
||||
"androidx.camera:camera-core:1.0.0-beta10",
|
||||
@@ -353,9 +348,9 @@ maven_install(
|
||||
"com.google.android.material:material:aar:1.0.0-rc01",
|
||||
"com.google.auto.value:auto-value:1.8.1",
|
||||
"com.google.auto.value:auto-value-annotations:1.8.1",
|
||||
"com.google.code.findbugs:jsr305:3.0.2",
|
||||
"com.google.flogger:flogger-system-backend:0.3.1",
|
||||
"com.google.flogger:flogger:0.3.1",
|
||||
"com.google.code.findbugs:jsr305:latest.release",
|
||||
"com.google.flogger:flogger-system-backend:latest.release",
|
||||
"com.google.flogger:flogger:latest.release",
|
||||
"com.google.guava:guava:27.0.1-android",
|
||||
"com.google.guava:listenablefuture:1.0",
|
||||
"junit:junit:4.12",
|
||||
@@ -383,9 +378,9 @@ http_archive(
|
||||
)
|
||||
|
||||
# Tensorflow repo should always go after the other external dependencies.
|
||||
# 2021-06-07
|
||||
_TENSORFLOW_GIT_COMMIT = "700533808e6016dc458bb2eeecfca4babfc482ec"
|
||||
_TENSORFLOW_SHA256 = "b6edd7f4039bfc19f3e77594ecff558ba620091d0dc48181484b3d9085026126"
|
||||
# 2021-07-29
|
||||
_TENSORFLOW_GIT_COMMIT = "52a2905cbc21034766c08041933053178c5d10e3"
|
||||
_TENSORFLOW_SHA256 = "06d4691bcdb700f3275fa0971a1585221c2b9f3dffe867963be565a6643d7f56"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
urls = [
|
||||
@@ -406,3 +401,18 @@ load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
|
||||
tf_workspace3()
|
||||
load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
|
||||
tf_workspace2()
|
||||
|
||||
# Edge TPU
|
||||
http_archive(
|
||||
name = "libedgetpu",
|
||||
sha256 = "14d5527a943a25bc648c28a9961f954f70ba4d79c0a9ca5ae226e1831d72fe80",
|
||||
strip_prefix = "libedgetpu-3164995622300286ef2bb14d7fdc2792dae045b7",
|
||||
urls = [
|
||||
"https://github.com/google-coral/libedgetpu/archive/3164995622300286ef2bb14d7fdc2792dae045b7.tar.gz"
|
||||
],
|
||||
)
|
||||
load("@libedgetpu//:workspace.bzl", "libedgetpu_dependencies")
|
||||
libedgetpu_dependencies()
|
||||
|
||||
load("@coral_crosstool//:configure.bzl", "cc_crosstool")
|
||||
cc_crosstool(name = "crosstool")
|
||||
|
||||
@@ -16,12 +16,14 @@ nav_order: 1
|
||||
|
||||
Please follow instructions below to build Android example apps in the supported
|
||||
MediaPipe [solutions](../solutions/solutions.md). To learn more about these
|
||||
example apps, start from [Hello World! on Android](./hello_world_android.md). To
|
||||
incorporate MediaPipe into an existing Android Studio project, see these
|
||||
[instructions](./android_archive_library.md) that use Android Archive (AAR) and
|
||||
Gradle.
|
||||
example apps, start from [Hello World! on Android](./hello_world_android.md).
|
||||
|
||||
## Building Android example apps
|
||||
To incorporate MediaPipe into Android Studio projects, see these
|
||||
[instructions](./android_solutions.md) to use the MediaPipe Android Solution
|
||||
APIs (currently in alpha) that are now available in
|
||||
[Google's Maven Repository](https://maven.google.com/web/index.html?#com.google.mediapipe).
|
||||
|
||||
## Building Android example apps with Bazel
|
||||
|
||||
### Prerequisite
|
||||
|
||||
@@ -51,16 +53,6 @@ $YOUR_INTENDED_API_LEVEL` in android_ndk_repository() and/or
|
||||
android_sdk_repository() in the
|
||||
[`WORKSPACE`](https://github.com/google/mediapipe/blob/master/WORKSPACE) file.
|
||||
|
||||
Please verify all the necessary packages are installed.
|
||||
|
||||
* Android SDK Platform API Level 28 or 29
|
||||
* Android SDK Build-Tools 28 or 29
|
||||
* Android SDK Platform-Tools 28 or 29
|
||||
* Android SDK Tools 26.1.1
|
||||
* Android NDK 19c or above
|
||||
|
||||
### Option 1: Build with Bazel in Command Line
|
||||
|
||||
Tip: You can run this
|
||||
[script](https://github.com/google/mediapipe/blob/master/build_android_examples.sh)
|
||||
to build (and install) all MediaPipe Android example apps.
|
||||
@@ -84,108 +76,3 @@ to build (and install) all MediaPipe Android example apps.
|
||||
```bash
|
||||
adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/handtrackinggpu.apk
|
||||
```
|
||||
|
||||
### Option 2: Build with Bazel in Android Studio
|
||||
|
||||
The MediaPipe project can be imported into Android Studio using the Bazel
|
||||
plugins. This allows the MediaPipe examples to be built and modified in Android
|
||||
Studio.
|
||||
|
||||
To incorporate MediaPipe into an existing Android Studio project, see these
|
||||
[instructions](./android_archive_library.md) that use Android Archive (AAR) and
|
||||
Gradle.
|
||||
|
||||
The steps below use Android Studio 3.5 to build and install a MediaPipe example
|
||||
app:
|
||||
|
||||
1. Install and launch Android Studio 3.5.
|
||||
|
||||
2. Select `Configure` -> `SDK Manager` -> `SDK Platforms`.
|
||||
|
||||
* Verify that Android SDK Platform API Level 28 or 29 is installed.
|
||||
* Take note of the Android SDK Location, e.g.,
|
||||
`/usr/local/home/Android/Sdk`.
|
||||
|
||||
3. Select `Configure` -> `SDK Manager` -> `SDK Tools`.
|
||||
|
||||
* Verify that Android SDK Build-Tools 28 or 29 is installed.
|
||||
* Verify that Android SDK Platform-Tools 28 or 29 is installed.
|
||||
* Verify that Android SDK Tools 26.1.1 is installed.
|
||||
* Verify that Android NDK 19c or above is installed.
|
||||
* Take note of the Android NDK Location, e.g.,
|
||||
`/usr/local/home/Android/Sdk/ndk-bundle` or
|
||||
`/usr/local/home/Android/Sdk/ndk/20.0.5594570`.
|
||||
|
||||
4. Set environment variables `$ANDROID_HOME` and `$ANDROID_NDK_HOME` to point
|
||||
to the installed SDK and NDK.
|
||||
|
||||
```bash
|
||||
export ANDROID_HOME=/usr/local/home/Android/Sdk
|
||||
|
||||
# If the NDK libraries are installed by a previous version of Android Studio, do
|
||||
export ANDROID_NDK_HOME=/usr/local/home/Android/Sdk/ndk-bundle
|
||||
# If the NDK libraries are installed by Android Studio 3.5, do
|
||||
export ANDROID_NDK_HOME=/usr/local/home/Android/Sdk/ndk/<version number>
|
||||
```
|
||||
|
||||
5. Select `Configure` -> `Plugins` to install `Bazel`.
|
||||
|
||||
6. On Linux, select `File` -> `Settings` -> `Bazel settings`. On macos, select
|
||||
`Android Studio` -> `Preferences` -> `Bazel settings`. Then, modify `Bazel
|
||||
binary location` to be the same as the output of `$ which bazel`.
|
||||
|
||||
7. Select `Import Bazel Project`.
|
||||
|
||||
* Select `Workspace`: `/path/to/mediapipe` and select `Next`.
|
||||
* Select `Generate from BUILD file`: `/path/to/mediapipe/BUILD` and select
|
||||
`Next`.
|
||||
* Modify `Project View` to be the following and select `Finish`.
|
||||
|
||||
```
|
||||
directories:
|
||||
# read project settings, e.g., .bazelrc
|
||||
.
|
||||
-mediapipe/objc
|
||||
-mediapipe/examples/ios
|
||||
|
||||
targets:
|
||||
//mediapipe/examples/android/...:all
|
||||
//mediapipe/java/...:all
|
||||
|
||||
android_sdk_platform: android-29
|
||||
|
||||
sync_flags:
|
||||
--host_crosstool_top=@bazel_tools//tools/cpp:toolchain
|
||||
```
|
||||
|
||||
8. Select `Bazel` -> `Sync` -> `Sync project with Build files`.
|
||||
|
||||
Note: Even after doing step 4, if you still see the error: `"no such package
|
||||
'@androidsdk//': Either the path attribute of android_sdk_repository or the
|
||||
ANDROID_HOME environment variable must be set."`, please modify the
|
||||
[`WORKSPACE`](https://github.com/google/mediapipe/blob/master/WORKSPACE)
|
||||
file to point to your SDK and NDK library locations, as below:
|
||||
|
||||
```
|
||||
android_sdk_repository(
|
||||
name = "androidsdk",
|
||||
path = "/path/to/android/sdk"
|
||||
)
|
||||
|
||||
android_ndk_repository(
|
||||
name = "androidndk",
|
||||
path = "/path/to/android/ndk"
|
||||
)
|
||||
```
|
||||
|
||||
9. Connect an Android device to the workstation.
|
||||
|
||||
10. Select `Run...` -> `Edit Configurations...`.
|
||||
|
||||
* Select `Templates` -> `Bazel Command`.
|
||||
* Enter Target Expression:
|
||||
`//mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu:handtrackinggpu`
|
||||
* Enter Bazel command: `mobile-install`.
|
||||
* Enter Bazel flags: `-c opt --config=android_arm64`.
|
||||
* Press the `[+]` button to add the new configuration.
|
||||
* Select `Run` to run the example app on the connected Android device.
|
||||
|
||||
@@ -3,7 +3,7 @@ layout: default
|
||||
title: MediaPipe Android Archive
|
||||
parent: MediaPipe on Android
|
||||
grand_parent: Getting Started
|
||||
nav_order: 2
|
||||
nav_order: 3
|
||||
---
|
||||
|
||||
# MediaPipe Android Archive
|
||||
@@ -113,9 +113,9 @@ each project.
|
||||
androidTestImplementation 'androidx.test.ext:junit:1.1.0'
|
||||
androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1'
|
||||
// MediaPipe deps
|
||||
implementation 'com.google.flogger:flogger:0.3.1'
|
||||
implementation 'com.google.flogger:flogger-system-backend:0.3.1'
|
||||
implementation 'com.google.code.findbugs:jsr305:3.0.2'
|
||||
implementation 'com.google.flogger:flogger:latest.release'
|
||||
implementation 'com.google.flogger:flogger-system-backend:latest.release'
|
||||
implementation 'com.google.code.findbugs:jsr305:latest.release'
|
||||
implementation 'com.google.guava:guava:27.0.1-android'
|
||||
implementation 'com.google.protobuf:protobuf-java:3.11.4'
|
||||
// CameraX core library
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
---
|
||||
layout: default
|
||||
title: Android Solutions
|
||||
parent: MediaPipe on Android
|
||||
grand_parent: Getting Started
|
||||
nav_order: 2
|
||||
---
|
||||
|
||||
# Android Solution APIs
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
Please follow instructions below to use the MediaPipe Solution APIs in Android
|
||||
Studio projects and build the Android example apps in the supported MediaPipe
|
||||
[solutions](../solutions/solutions.md).
|
||||
|
||||
## Integrate MediaPipe Android Solutions in Android Studio
|
||||
|
||||
MediaPipe Android Solution APIs (currently in alpha) are now available in
|
||||
[Google's Maven Repository](https://maven.google.com/web/index.html?#com.google.mediapipe).
|
||||
To incorporate MediaPipe Android Solutions into an Android Studio project, add
|
||||
the following into the project's Gradle dependencies:
|
||||
|
||||
```
|
||||
dependencies {
|
||||
// MediaPipe solution-core is the foundation of any MediaPipe solutions.
|
||||
implementation 'com.google.mediapipe:solution-core:latest.release'
|
||||
// Optional: MediaPipe Hands solution.
|
||||
implementation 'com.google.mediapipe:hands:latest.release'
|
||||
// Optional: MediaPipe FaceMesh solution.
|
||||
implementation 'com.google.mediapipe:facemesh:latest.release'
|
||||
// MediaPipe deps
|
||||
implementation 'com.google.flogger:flogger:latest.release'
|
||||
implementation 'com.google.flogger:flogger-system-backend:latest.release'
|
||||
implementation 'com.google.guava:guava:27.0.1-android'
|
||||
implementation 'com.google.protobuf:protobuf-java:3.11.4'
|
||||
// CameraX core library
|
||||
def camerax_version = "1.0.0-beta10"
|
||||
implementation "androidx.camera:camera-core:$camerax_version"
|
||||
implementation "androidx.camera:camera-camera2:$camerax_version"
|
||||
implementation "androidx.camera:camera-lifecycle:$camerax_version"
|
||||
}
|
||||
```
|
||||
|
||||
See the detailed solutions API usage examples for different use cases in the
|
||||
solution example apps'
|
||||
[source code](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions).
|
||||
If the prebuilt maven packages are not sufficient, building the MediaPipe
|
||||
Android archive library locally by following these
|
||||
[instructions](./android_archive_library.md).
|
||||
|
||||
## Build solution example apps in Android Studio
|
||||
|
||||
1. Open Android Studio Arctic Fox on Linux, macOS, or Windows.
|
||||
|
||||
2. Import mediapipe/examples/android/solutions directory into Android Studio.
|
||||
|
||||
|
||||
|
||||
3. For Windows users, run `create_win_symlinks.bat` as administrator to create
|
||||
res directory symlinks.
|
||||
|
||||
|
||||
|
||||
4. Select "File" -> "Sync Project with Gradle Files" to sync project.
|
||||
|
||||
5. Run solution example app in Android Studio.
|
||||
|
||||
|
||||
|
||||
6. (Optional) Run solutions on CPU.
|
||||
|
||||
MediaPipe solution example apps run the pipeline and the model inference on
|
||||
GPU by default. If needed, for example to run the apps on Android Emulator,
|
||||
set the `RUN_ON_GPU` boolean variable to `false` in the app's
|
||||
MainActivity.java to run the pipeline and the model inference on CPU.
|
||||
@@ -31,8 +31,8 @@ stream on an Android device.
|
||||
|
||||
## Setup
|
||||
|
||||
1. Install MediaPipe on your system, see [MediaPipe installation guide] for
|
||||
details.
|
||||
1. Install MediaPipe on your system, see
|
||||
[MediaPipe installation guide](./install.md) for details.
|
||||
2. Install Android Development SDK and Android NDK. See how to do so also in
|
||||
[MediaPipe installation guide].
|
||||
3. Enable [developer options] on your Android device.
|
||||
@@ -770,7 +770,6 @@ If you ran into any issues, please see the full code of the tutorial
|
||||
[`ExternalTextureConverter`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java
|
||||
[`FrameLayout`]:https://developer.android.com/reference/android/widget/FrameLayout
|
||||
[`FrameProcessor`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/FrameProcessor.java
|
||||
[MediaPipe installation guide]:./install.md
|
||||
[`PermissionHelper`]: https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/PermissionHelper.java
|
||||
[`SurfaceHolder.Callback`]:https://developer.android.com/reference/android/view/SurfaceHolder.Callback.html
|
||||
[`SurfaceView`]:https://developer.android.com/reference/android/view/SurfaceView
|
||||
|
||||
@@ -31,8 +31,8 @@ stream on an iOS device.
|
||||
|
||||
## Setup
|
||||
|
||||
1. Install MediaPipe on your system, see [MediaPipe installation guide] for
|
||||
details.
|
||||
1. Install MediaPipe on your system, see
|
||||
[MediaPipe installation guide](./install.md) for details.
|
||||
2. Setup your iOS device for development.
|
||||
3. Setup [Bazel] on your system to build and deploy the iOS app.
|
||||
|
||||
@@ -560,6 +560,5 @@ appropriate `BUILD` file dependencies for the edge detection graph.
|
||||
|
||||
[Bazel]:https://bazel.build/
|
||||
[`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/edge_detection/edge_detection_mobile_gpu.pbtxt
|
||||
[MediaPipe installation guide]:./install.md
|
||||
[common]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/common)
|
||||
[helloworld]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/helloworld)
|
||||
|
||||
+158
-73
@@ -43,104 +43,189 @@ install --user six`.
|
||||
|
||||
3. Install OpenCV and FFmpeg.
|
||||
|
||||
Option 1. Use package manager tool to install the pre-compiled OpenCV
|
||||
libraries. FFmpeg will be installed via libopencv-video-dev.
|
||||
**Option 1**. Use package manager tool to install the pre-compiled OpenCV
|
||||
libraries. FFmpeg will be installed via `libopencv-video-dev`.
|
||||
|
||||
Note: Debian 9 and Ubuntu 16.04 provide OpenCV 2.4.9. You may want to take
|
||||
option 2 or 3 to install OpenCV 3 or above.
|
||||
OS | OpenCV
|
||||
-------------------- | ------
|
||||
Debian 9 (stretch) | 2.4
|
||||
Debian 10 (buster) | 3.2
|
||||
Debian 11 (bullseye) | 4.5
|
||||
Ubuntu 16.04 LTS | 2.4
|
||||
Ubuntu 18.04 LTS | 3.2
|
||||
Ubuntu 20.04 LTS | 4.2
|
||||
Ubuntu 20.04 LTS | 4.2
|
||||
Ubuntu 21.04 | 4.5
|
||||
|
||||
```bash
|
||||
$ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
|
||||
libopencv-calib3d-dev libopencv-features2d-dev \
|
||||
libopencv-imgproc-dev libopencv-video-dev
|
||||
$ sudo apt-get install -y \
|
||||
libopencv-core-dev \
|
||||
libopencv-highgui-dev \
|
||||
libopencv-calib3d-dev \
|
||||
libopencv-features2d-dev \
|
||||
libopencv-imgproc-dev \
|
||||
libopencv-video-dev
|
||||
```
|
||||
|
||||
Debian 9 and Ubuntu 18.04 install the packages in
|
||||
`/usr/lib/x86_64-linux-gnu`. MediaPipe's [`opencv_linux.BUILD`] and
|
||||
[`ffmpeg_linux.BUILD`] are configured for this library path. Ubuntu 20.04
|
||||
may install the OpenCV and FFmpeg packages in `/usr/local`, Please follow
|
||||
the option 3 below to modify the [`WORKSPACE`], [`opencv_linux.BUILD`] and
|
||||
[`ffmpeg_linux.BUILD`] files accordingly.
|
||||
|
||||
Moreover, for Nvidia Jetson and Raspberry Pi devices with ARM Ubuntu, the
|
||||
library path needs to be modified like the following:
|
||||
MediaPipe's [`opencv_linux.BUILD`] and [`WORKSPACE`] are already configured
|
||||
for OpenCV 2/3 and should work correctly on any architecture:
|
||||
|
||||
```bash
|
||||
sed -i "s/x86_64-linux-gnu/aarch64-linux-gnu/g" third_party/opencv_linux.BUILD
|
||||
# WORKSPACE
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
# opencv_linux.BUILD for OpenCV 2/3 installed from Debian package
|
||||
cc_library(
|
||||
name = "opencv",
|
||||
linkopts = [
|
||||
"-l:libopencv_core.so",
|
||||
"-l:libopencv_calib3d.so",
|
||||
"-l:libopencv_features2d.so",
|
||||
"-l:libopencv_highgui.so",
|
||||
"-l:libopencv_imgcodecs.so",
|
||||
"-l:libopencv_imgproc.so",
|
||||
"-l:libopencv_video.so",
|
||||
"-l:libopencv_videoio.so",
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
Option 2. Run [`setup_opencv.sh`] to automatically build OpenCV from source
|
||||
and modify MediaPipe's OpenCV config.
|
||||
For OpenCV 4 you need to modify [`opencv_linux.BUILD`] taking into account
|
||||
current architecture:
|
||||
|
||||
Option 3. Follow OpenCV's
|
||||
```bash
|
||||
# WORKSPACE
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
# opencv_linux.BUILD for OpenCV 4 installed from Debian package
|
||||
cc_library(
|
||||
name = "opencv",
|
||||
hdrs = glob([
|
||||
# Uncomment according to your multiarch value (gcc -print-multiarch):
|
||||
# "include/aarch64-linux-gnu/opencv4/opencv2/cvconfig.h",
|
||||
# "include/arm-linux-gnueabihf/opencv4/opencv2/cvconfig.h",
|
||||
# "include/x86_64-linux-gnu/opencv4/opencv2/cvconfig.h",
|
||||
"include/opencv4/opencv2/**/*.h*",
|
||||
]),
|
||||
includes = [
|
||||
# Uncomment according to your multiarch value (gcc -print-multiarch):
|
||||
# "include/aarch64-linux-gnu/opencv4/",
|
||||
# "include/arm-linux-gnueabihf/opencv4/",
|
||||
# "include/x86_64-linux-gnu/opencv4/",
|
||||
"include/opencv4/",
|
||||
],
|
||||
linkopts = [
|
||||
"-l:libopencv_core.so",
|
||||
"-l:libopencv_calib3d.so",
|
||||
"-l:libopencv_features2d.so",
|
||||
"-l:libopencv_highgui.so",
|
||||
"-l:libopencv_imgcodecs.so",
|
||||
"-l:libopencv_imgproc.so",
|
||||
"-l:libopencv_video.so",
|
||||
"-l:libopencv_videoio.so",
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
**Option 2**. Run [`setup_opencv.sh`] to automatically build OpenCV from
|
||||
source and modify MediaPipe's OpenCV config. This option will do all steps
|
||||
defined in Option 3 automatically.
|
||||
|
||||
**Option 3**. Follow OpenCV's
|
||||
[documentation](https://docs.opencv.org/3.4.6/d7/d9f/tutorial_linux_install.html)
|
||||
to manually build OpenCV from source code.
|
||||
|
||||
Note: You may need to modify [`WORKSPACE`], [`opencv_linux.BUILD`] and
|
||||
[`ffmpeg_linux.BUILD`] to point MediaPipe to your own OpenCV and FFmpeg
|
||||
libraries. For example if OpenCV and FFmpeg are both manually installed in
|
||||
"/usr/local/", you will need to update: (1) the "linux_opencv" and
|
||||
"linux_ffmpeg" new_local_repository rules in [`WORKSPACE`], (2) the "opencv"
|
||||
cc_library rule in [`opencv_linux.BUILD`], and (3) the "libffmpeg"
|
||||
cc_library rule in [`ffmpeg_linux.BUILD`]. These 3 changes are shown below:
|
||||
You may need to modify [`WORKSPACE`] and [`opencv_linux.BUILD`] to point
|
||||
MediaPipe to your own OpenCV libraries. Assume OpenCV would be installed to
|
||||
`/usr/local/` which is recommended by default.
|
||||
|
||||
OpenCV 2/3 setup:
|
||||
|
||||
```bash
|
||||
# WORKSPACE
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr/local",
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr/local",
|
||||
)
|
||||
|
||||
# opencv_linux.BUILD for OpenCV 2/3 installed to /usr/local
|
||||
cc_library(
|
||||
name = "opencv",
|
||||
linkopts = [
|
||||
"-L/usr/local/lib",
|
||||
"-l:libopencv_core.so",
|
||||
"-l:libopencv_calib3d.so",
|
||||
"-l:libopencv_features2d.so",
|
||||
"-l:libopencv_highgui.so",
|
||||
"-l:libopencv_imgcodecs.so",
|
||||
"-l:libopencv_imgproc.so",
|
||||
"-l:libopencv_video.so",
|
||||
"-l:libopencv_videoio.so",
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
OpenCV 4 setup:
|
||||
|
||||
```bash
|
||||
# WORKSPACE
|
||||
new_local_repository(
|
||||
name = "linux_ffmpeg",
|
||||
build_file = "@//third_party:ffmpeg_linux.BUILD",
|
||||
path = "/usr/local",
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr/local",
|
||||
)
|
||||
|
||||
# opencv_linux.BUILD for OpenCV 4 installed to /usr/local
|
||||
cc_library(
|
||||
name = "opencv",
|
||||
srcs = glob(
|
||||
[
|
||||
"lib/libopencv_core.so",
|
||||
"lib/libopencv_highgui.so",
|
||||
"lib/libopencv_imgcodecs.so",
|
||||
"lib/libopencv_imgproc.so",
|
||||
"lib/libopencv_video.so",
|
||||
"lib/libopencv_videoio.so",
|
||||
],
|
||||
),
|
||||
hdrs = glob([
|
||||
# For OpenCV 3.x
|
||||
"include/opencv2/**/*.h*",
|
||||
# For OpenCV 4.x
|
||||
# "include/opencv4/opencv2/**/*.h*",
|
||||
]),
|
||||
includes = [
|
||||
# For OpenCV 3.x
|
||||
"include/",
|
||||
# For OpenCV 4.x
|
||||
# "include/opencv4/",
|
||||
],
|
||||
linkstatic = 1,
|
||||
visibility = ["//visibility:public"],
|
||||
name = "opencv",
|
||||
hdrs = glob([
|
||||
"include/opencv4/opencv2/**/*.h*",
|
||||
]),
|
||||
includes = [
|
||||
"include/opencv4/",
|
||||
],
|
||||
linkopts = [
|
||||
"-L/usr/local/lib",
|
||||
"-l:libopencv_core.so",
|
||||
"-l:libopencv_calib3d.so",
|
||||
"-l:libopencv_features2d.so",
|
||||
"-l:libopencv_highgui.so",
|
||||
"-l:libopencv_imgcodecs.so",
|
||||
"-l:libopencv_imgproc.so",
|
||||
"-l:libopencv_video.so",
|
||||
"-l:libopencv_videoio.so",
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
Current FFmpeg setup is defined in [`ffmpeg_linux.BUILD`] and should work
|
||||
for any architecture:
|
||||
|
||||
```bash
|
||||
# WORKSPACE
|
||||
new_local_repository(
|
||||
name = "linux_ffmpeg",
|
||||
build_file = "@//third_party:ffmpeg_linux.BUILD",
|
||||
path = "/usr"
|
||||
)
|
||||
|
||||
# ffmpeg_linux.BUILD for FFmpeg installed from Debian package
|
||||
cc_library(
|
||||
name = "libffmpeg",
|
||||
srcs = glob(
|
||||
[
|
||||
"lib/libav*.so",
|
||||
],
|
||||
),
|
||||
hdrs = glob(["include/libav*/*.h"]),
|
||||
includes = ["include"],
|
||||
linkopts = [
|
||||
"-lavcodec",
|
||||
"-lavformat",
|
||||
"-lavutil",
|
||||
],
|
||||
linkstatic = 1,
|
||||
visibility = ["//visibility:public"],
|
||||
name = "libffmpeg",
|
||||
linkopts = [
|
||||
"-l:libavcodec.so",
|
||||
"-l:libavformat.so",
|
||||
"-l:libavutil.so",
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
|
||||
@@ -22,12 +22,23 @@ Solution | NPM Package | Example
|
||||
[Face Detection][Fd-pg] | [@mediapipe/face_detection][Fd-npm] | [mediapipe.dev/demo/face_detection][Fd-demo]
|
||||
[Hands][H-pg] | [@mediapipe/hands][H-npm] | [mediapipe.dev/demo/hands][H-demo]
|
||||
[Holistic][Ho-pg] | [@mediapipe/holistic][Ho-npm] | [mediapipe.dev/demo/holistic][Ho-demo]
|
||||
[Objectron][Ob-pg] | [@mediapipe/objectron][Ob-npm] | [mediapipe.dev/demo/objectron][Ob-demo]
|
||||
[Pose][P-pg] | [@mediapipe/pose][P-npm] | [mediapipe.dev/demo/pose][P-demo]
|
||||
[Selfie Segmentation][S-pg] | [@mediapipe/selfie_segmentation][S-npm] | [mediapipe.dev/demo/selfie_segmentation][S-demo]
|
||||
|
||||
Click on a solution link above for more information, including API and code
|
||||
snippets.
|
||||
|
||||
### Supported plaforms:
|
||||
|
||||
| Browser | Platform | Notes |
|
||||
| ------- | ----------------------- | -------------------------------------- |
|
||||
| Chrome | Android / Windows / Mac | Pixel 4 and older unsupported. Fuschia |
|
||||
| | | unsupported. |
|
||||
| Chrome | iOS | Camera unavailable in Chrome on iOS. |
|
||||
| Safari | iPad/iPhone/Mac | iOS and Safari on iPad / iPhone / |
|
||||
| | | MacBook |
|
||||
|
||||
The quickest way to get acclimated is to look at the examples above. Each demo
|
||||
has a link to a [CodePen][codepen] so that you can edit the code and try it
|
||||
yourself. We have included a number of utility packages to help you get started:
|
||||
@@ -67,33 +78,24 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
|
||||
[F-pg]: ../solutions/face_mesh#javascript-solution-api
|
||||
[Fd-pg]: ../solutions/face_detection#javascript-solution-api
|
||||
[H-pg]: ../solutions/hands#javascript-solution-api
|
||||
[Ob-pg]: ../solutions/objectron#javascript-solution-api
|
||||
[P-pg]: ../solutions/pose#javascript-solution-api
|
||||
[S-pg]: ../solutions/selfie_segmentation#javascript-solution-api
|
||||
[Ho-npm]: https://www.npmjs.com/package/@mediapipe/holistic
|
||||
[F-npm]: https://www.npmjs.com/package/@mediapipe/face_mesh
|
||||
[Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection
|
||||
[H-npm]: https://www.npmjs.com/package/@mediapipe/hands
|
||||
[Ob-npm]: https://www.npmjs.com/package/@mediapipe/objectron
|
||||
[P-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[S-npm]: https://www.npmjs.com/package/@mediapipe/selfie_segmentation
|
||||
[draw-npm]: https://www.npmjs.com/package/@mediapipe/drawing_utils
|
||||
[cam-npm]: https://www.npmjs.com/package/@mediapipe/camera_utils
|
||||
[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/control_utils
|
||||
[Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic
|
||||
[F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh
|
||||
[Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection
|
||||
[H-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/hands
|
||||
[P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/pose
|
||||
[P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/selfie_segmentation
|
||||
[Ho-pen]: https://code.mediapipe.dev/codepen/holistic
|
||||
[F-pen]: https://code.mediapipe.dev/codepen/face_mesh
|
||||
[Fd-pen]: https://code.mediapipe.dev/codepen/face_detection
|
||||
[H-pen]: https://code.mediapipe.dev/codepen/hands
|
||||
[P-pen]: https://code.mediapipe.dev/codepen/pose
|
||||
[S-pen]: https://code.mediapipe.dev/codepen/selfie_segmentation
|
||||
[Ho-demo]: https://mediapipe.dev/demo/holistic
|
||||
[F-demo]: https://mediapipe.dev/demo/face_mesh
|
||||
[Fd-demo]: https://mediapipe.dev/demo/face_detection
|
||||
[H-demo]: https://mediapipe.dev/demo/hands
|
||||
[Ob-demo]: https://mediapipe.dev/demo/objectron
|
||||
[P-demo]: https://mediapipe.dev/demo/pose
|
||||
[S-demo]: https://mediapipe.dev/demo/selfie_segmentation
|
||||
[npm]: https://www.npmjs.com/package/@mediapipe
|
||||
|
||||
@@ -74,7 +74,7 @@ Mapping\[str, Packet\] | std::map<std::string, Packet> | create_st
|
||||
np.ndarray<br>(cv.mat and PIL.Image) | mp::ImageFrame | create_image_frame(<br> format=ImageFormat.SRGB,<br> data=mat) | get_image_frame(packet)
|
||||
np.ndarray | mp::Matrix | create_matrix(data) | get_matrix(packet)
|
||||
Google Proto Message | Google Proto Message | create_proto(proto) | get_proto(packet)
|
||||
List\[Proto\] | std::vector\<Proto\> | create_proto_vector(proto_list) | get_proto_list(packet)
|
||||
List\[Proto\] | std::vector\<Proto\> | n/a | get_proto_list(packet)
|
||||
|
||||
It's not uncommon that users create custom C++ classes and and send those into
|
||||
the graphs and calculators. To allow the custom classes to be used in Python
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 128 KiB |
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 258 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 51 KiB |
+8
-1
@@ -45,7 +45,7 @@ Hair Segmentation
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
@@ -79,6 +79,13 @@ run code search using
|
||||
|
||||
## Publications
|
||||
|
||||
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
||||
in Google Developers Blog
|
||||
* [Prosthesis control via Mirru App using MediaPipe hand tracking](https://developers.googleblog.com/2021/05/control-your-mirru-prosthesis-with-mediapipe-hand-tracking.html)
|
||||
in Google Developers Blog
|
||||
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
||||
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
||||
in Google Developers Blog
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
in Google AI Blog
|
||||
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
||||
|
||||
+217
-6
@@ -278,6 +278,7 @@ Supported configuration options:
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_drawing_styles = mp.solutions.drawing_styles
|
||||
mp_face_mesh = mp.solutions.face_mesh
|
||||
|
||||
# For static images:
|
||||
@@ -301,9 +302,17 @@ with mp_face_mesh.FaceMesh(
|
||||
mp_drawing.draw_landmarks(
|
||||
image=annotated_image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACE_CONNECTIONS,
|
||||
landmark_drawing_spec=drawing_spec,
|
||||
connection_drawing_spec=drawing_spec)
|
||||
connections=mp_face_mesh.FACEMESH_TESSELATION,
|
||||
landmark_drawing_spec=None,
|
||||
connection_drawing_spec=mp_drawing_styles
|
||||
.get_default_face_mesh_tesselation_style())
|
||||
mp_drawing.draw_landmarks(
|
||||
image=annotated_image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACEMESH_CONTOURS,
|
||||
landmark_drawing_spec=None,
|
||||
connection_drawing_spec=mp_drawing_styles
|
||||
.get_default_face_mesh_contours_style())
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
|
||||
# For webcam input:
|
||||
@@ -335,9 +344,17 @@ with mp_face_mesh.FaceMesh(
|
||||
mp_drawing.draw_landmarks(
|
||||
image=image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACE_CONNECTIONS,
|
||||
landmark_drawing_spec=drawing_spec,
|
||||
connection_drawing_spec=drawing_spec)
|
||||
connections=mp_face_mesh.FACEMESH_TESSELATION,
|
||||
landmark_drawing_spec=None,
|
||||
connection_drawing_spec=mp_drawing_styles
|
||||
.get_default_face_mesh_tesselation_style())
|
||||
mp_drawing.draw_landmarks(
|
||||
image=image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACEMESH_CONTOURS,
|
||||
landmark_drawing_spec=None,
|
||||
connection_drawing_spec=mp_drawing_styles
|
||||
.get_default_face_mesh_contours_style())
|
||||
cv2.imshow('MediaPipe FaceMesh', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
@@ -423,6 +440,200 @@ camera.start();
|
||||
</script>
|
||||
```
|
||||
|
||||
### Android Solution API
|
||||
|
||||
Please first follow general
|
||||
[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
|
||||
to add MediaPipe Gradle dependencies, then try the FaceMash solution API in the
|
||||
companion
|
||||
[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh)
|
||||
following
|
||||
[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
|
||||
and learn more in the usage example below.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
* [staticImageMode](#static_image_mode)
|
||||
* [maxNumFaces](#max_num_faces)
|
||||
* runOnGpu: Run the pipeline and the model inference on GPU or CPU.
|
||||
|
||||
#### Camera Input
|
||||
|
||||
```java
|
||||
// For camera input and result rendering with OpenGL.
|
||||
FaceMeshOptions faceMeshOptions =
|
||||
FaceMeshOptions.builder()
|
||||
.setMode(FaceMeshOptions.STREAMING_MODE) // API soon to become
|
||||
.setMaxNumFaces(1) // setStaticImageMode(false)
|
||||
.setRunOnGpu(true).build();
|
||||
FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
|
||||
facemesh.setErrorListener(
|
||||
(message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
|
||||
|
||||
// Initializes a new CameraInput instance and connects it to MediaPipe FaceMesh.
|
||||
CameraInput cameraInput = new CameraInput(this);
|
||||
cameraInput.setNewFrameListener(
|
||||
textureFrame -> facemesh.send(textureFrame));
|
||||
|
||||
// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
|
||||
// that provides the interfaces to run user-defined OpenGL rendering code.
|
||||
// See mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultGlRenderer.java
|
||||
// as an example.
|
||||
SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
|
||||
new SolutionGlSurfaceView<>(
|
||||
this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
|
||||
glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
|
||||
glSurfaceView.setRenderInputImage(true);
|
||||
|
||||
facemesh.setResultListener(
|
||||
faceMeshResult -> {
|
||||
NormalizedLandmark noseLandmark =
|
||||
result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
|
||||
Log.i(
|
||||
TAG,
|
||||
String.format(
|
||||
"MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
|
||||
noseLandmark.getX(), noseLandmark.getY()));
|
||||
// Request GL rendering.
|
||||
glSurfaceView.setRenderData(faceMeshResult);
|
||||
glSurfaceView.requestRender();
|
||||
});
|
||||
|
||||
// The runnable to start camera after the GLSurfaceView is attached.
|
||||
glSurfaceView.post(
|
||||
() ->
|
||||
cameraInput.start(
|
||||
this,
|
||||
facemesh.getGlContext(),
|
||||
CameraInput.CameraFacing.FRONT,
|
||||
glSurfaceView.getWidth(),
|
||||
glSurfaceView.getHeight()));
|
||||
```
|
||||
|
||||
#### Image Input
|
||||
|
||||
```java
|
||||
// For reading images from gallery and drawing the output in an ImageView.
|
||||
FaceMeshOptions faceMeshOptions =
|
||||
FaceMeshOptions.builder()
|
||||
.setMode(FaceMeshOptions.STATIC_IMAGE_MODE) // API soon to become
|
||||
.setMaxNumFaces(1) // setStaticImageMode(true)
|
||||
.setRunOnGpu(true).build();
|
||||
FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
|
||||
|
||||
// Connects MediaPipe FaceMesh to the user-defined ImageView instance that allows
|
||||
// users to have the custom drawing of the output landmarks on it.
|
||||
// See mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultImageView.java
|
||||
// as an example.
|
||||
FaceMeshResultImageView imageView = new FaceMeshResultImageView(this);
|
||||
facemesh.setResultListener(
|
||||
faceMeshResult -> {
|
||||
int width = faceMeshResult.inputBitmap().getWidth();
|
||||
int height = faceMeshResult.inputBitmap().getHeight();
|
||||
NormalizedLandmark noseLandmark =
|
||||
result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
|
||||
Log.i(
|
||||
TAG,
|
||||
String.format(
|
||||
"MediaPipe FaceMesh nose coordinates (pixel values): x=%f, y=%f",
|
||||
noseLandmark.getX() * width, noseLandmark.getY() * height));
|
||||
// Request canvas drawing.
|
||||
imageView.setFaceMeshResult(faceMeshResult);
|
||||
runOnUiThread(() -> imageView.update());
|
||||
});
|
||||
facemesh.setErrorListener(
|
||||
(message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
|
||||
|
||||
// ActivityResultLauncher to get an image from the gallery as Bitmap.
|
||||
ActivityResultLauncher<Intent> imageGetter =
|
||||
registerForActivityResult(
|
||||
new ActivityResultContracts.StartActivityForResult(),
|
||||
result -> {
|
||||
Intent resultIntent = result.getData();
|
||||
if (resultIntent != null && result.getResultCode() == RESULT_OK) {
|
||||
Bitmap bitmap = null;
|
||||
try {
|
||||
bitmap =
|
||||
MediaStore.Images.Media.getBitmap(
|
||||
this.getContentResolver(), resultIntent.getData());
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "Bitmap reading error:" + e);
|
||||
}
|
||||
if (bitmap != null) {
|
||||
facemesh.send(bitmap);
|
||||
}
|
||||
}
|
||||
});
|
||||
Intent gallery = new Intent(
|
||||
Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
|
||||
imageGetter.launch(gallery);
|
||||
```
|
||||
|
||||
#### Video Input
|
||||
|
||||
```java
|
||||
// For video input and result rendering with OpenGL.
|
||||
FaceMeshOptions faceMeshOptions =
|
||||
FaceMeshOptions.builder()
|
||||
.setMode(FaceMeshOptions.STREAMING_MODE) // API soon to become
|
||||
.setMaxNumFaces(1) // setStaticImageMode(false)
|
||||
.setRunOnGpu(true).build();
|
||||
FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
|
||||
facemesh.setErrorListener(
|
||||
(message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
|
||||
|
||||
// Initializes a new VideoInput instance and connects it to MediaPipe FaceMesh.
|
||||
VideoInput videoInput = new VideoInput(this);
|
||||
videoInput.setNewFrameListener(
|
||||
textureFrame -> facemesh.send(textureFrame));
|
||||
|
||||
// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
|
||||
// that provides the interfaces to run user-defined OpenGL rendering code.
|
||||
// See mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultGlRenderer.java
|
||||
// as an example.
|
||||
SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
|
||||
new SolutionGlSurfaceView<>(
|
||||
this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
|
||||
glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
|
||||
glSurfaceView.setRenderInputImage(true);
|
||||
|
||||
facemesh.setResultListener(
|
||||
faceMeshResult -> {
|
||||
NormalizedLandmark noseLandmark =
|
||||
result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
|
||||
Log.i(
|
||||
TAG,
|
||||
String.format(
|
||||
"MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
|
||||
noseLandmark.getX(), noseLandmark.getY()));
|
||||
// Request GL rendering.
|
||||
glSurfaceView.setRenderData(faceMeshResult);
|
||||
glSurfaceView.requestRender();
|
||||
});
|
||||
|
||||
ActivityResultLauncher<Intent> videoGetter =
|
||||
registerForActivityResult(
|
||||
new ActivityResultContracts.StartActivityForResult(),
|
||||
result -> {
|
||||
Intent resultIntent = result.getData();
|
||||
if (resultIntent != null) {
|
||||
if (result.getResultCode() == RESULT_OK) {
|
||||
glSurfaceView.post(
|
||||
() ->
|
||||
videoInput.start(
|
||||
this,
|
||||
resultIntent.getData(),
|
||||
facemesh.getGlContext(),
|
||||
glSurfaceView.getWidth(),
|
||||
glSurfaceView.getHeight()));
|
||||
}
|
||||
}
|
||||
});
|
||||
Intent gallery =
|
||||
new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
|
||||
videoGetter.launch(gallery);
|
||||
```
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
|
||||
+205
-2
@@ -219,6 +219,7 @@ Supported configuration options:
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_drawing_styles = mp.solutions.drawing_styles
|
||||
mp_hands = mp.solutions.hands
|
||||
|
||||
# For static images:
|
||||
@@ -248,7 +249,11 @@ with mp_hands.Hands(
|
||||
f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * image_height})'
|
||||
)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
annotated_image,
|
||||
hand_landmarks,
|
||||
mp_hands.HAND_CONNECTIONS,
|
||||
mp_drawing_styles.get_default_hand_landmarks_style(),
|
||||
mp_drawing_styles.get_default_hand_connections_style())
|
||||
cv2.imwrite(
|
||||
'/tmp/annotated_image' + str(idx) + '.png', cv2.flip(annotated_image, 1))
|
||||
|
||||
@@ -278,7 +283,11 @@ with mp_hands.Hands(
|
||||
if results.multi_hand_landmarks:
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
mp_drawing.draw_landmarks(
|
||||
image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
image,
|
||||
hand_landmarks,
|
||||
mp_hands.HAND_CONNECTIONS,
|
||||
mp_drawing_styles.get_default_hand_landmarks_style(),
|
||||
mp_drawing_styles.get_default_hand_connections_style())
|
||||
cv2.imshow('MediaPipe Hands', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
@@ -359,6 +368,200 @@ camera.start();
|
||||
</script>
|
||||
```
|
||||
|
||||
### Android Solution API
|
||||
|
||||
Please first follow general
|
||||
[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
|
||||
to add MediaPipe Gradle dependencies, then try the Hands solution API in the
|
||||
companion
|
||||
[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands)
|
||||
following
|
||||
[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
|
||||
and learn more in usage example below.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
* [staticImageMode](#static_image_mode)
|
||||
* [maxNumHands](#max_num_hands)
|
||||
* runOnGpu: Run the pipeline and the model inference on GPU or CPU.
|
||||
|
||||
#### Camera Input
|
||||
|
||||
```java
|
||||
// For camera input and result rendering with OpenGL.
|
||||
HandsOptions handsOptions =
|
||||
HandsOptions.builder()
|
||||
.setMode(HandsOptions.STREAMING_MODE) // API soon to become
|
||||
.setMaxNumHands(1) // setStaticImageMode(false)
|
||||
.setRunOnGpu(true).build();
|
||||
Hands hands = new Hands(this, handsOptions);
|
||||
hands.setErrorListener(
|
||||
(message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
|
||||
|
||||
// Initializes a new CameraInput instance and connects it to MediaPipe Hands.
|
||||
CameraInput cameraInput = new CameraInput(this);
|
||||
cameraInput.setNewFrameListener(
|
||||
textureFrame -> hands.send(textureFrame));
|
||||
|
||||
// Initializes a new GlSurfaceView with a ResultGlRenderer<HandsResult> instance
|
||||
// that provides the interfaces to run user-defined OpenGL rendering code.
|
||||
// See mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java
|
||||
// as an example.
|
||||
SolutionGlSurfaceView<HandsResult> glSurfaceView =
|
||||
new SolutionGlSurfaceView<>(
|
||||
this, hands.getGlContext(), hands.getGlMajorVersion());
|
||||
glSurfaceView.setSolutionResultRenderer(new HandsResultGlRenderer());
|
||||
glSurfaceView.setRenderInputImage(true);
|
||||
|
||||
hands.setResultListener(
|
||||
handsResult -> {
|
||||
NormalizedLandmark wristLandmark = Hands.getHandLandmark(
|
||||
handsResult, 0, HandLandmark.WRIST);
|
||||
Log.i(
|
||||
TAG,
|
||||
String.format(
|
||||
"MediaPipe Hand wrist normalized coordinates (value range: [0, 1]): x=%f, y=%f",
|
||||
wristLandmark.getX(), wristLandmark.getY()));
|
||||
// Request GL rendering.
|
||||
glSurfaceView.setRenderData(handsResult);
|
||||
glSurfaceView.requestRender();
|
||||
});
|
||||
|
||||
// The runnable to start camera after the GLSurfaceView is attached.
|
||||
glSurfaceView.post(
|
||||
() ->
|
||||
cameraInput.start(
|
||||
this,
|
||||
hands.getGlContext(),
|
||||
CameraInput.CameraFacing.FRONT,
|
||||
glSurfaceView.getWidth(),
|
||||
glSurfaceView.getHeight()));
|
||||
```
|
||||
|
||||
#### Image Input
|
||||
|
||||
```java
|
||||
// For reading images from gallery and drawing the output in an ImageView.
|
||||
HandsOptions handsOptions =
|
||||
HandsOptions.builder()
|
||||
.setMode(HandsOptions.STATIC_IMAGE_MODE) // API soon to become
|
||||
.setMaxNumHands(1) // setStaticImageMode(true)
|
||||
.setRunOnGpu(true).build();
|
||||
Hands hands = new Hands(this, handsOptions);
|
||||
|
||||
// Connects MediaPipe Hands to the user-defined ImageView instance that allows
|
||||
// users to have the custom drawing of the output landmarks on it.
|
||||
// See mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java
|
||||
// as an example.
|
||||
HandsResultImageView imageView = new HandsResultImageView(this);
|
||||
hands.setResultListener(
|
||||
handsResult -> {
|
||||
int width = handsResult.inputBitmap().getWidth();
|
||||
int height = handsResult.inputBitmap().getHeight();
|
||||
NormalizedLandmark wristLandmark = Hands.getHandLandmark(
|
||||
handsResult, 0, HandLandmark.WRIST);
|
||||
Log.i(
|
||||
TAG,
|
||||
String.format(
|
||||
"MediaPipe Hand wrist coordinates (pixel values): x=%f, y=%f",
|
||||
wristLandmark.getX() * width, wristLandmark.getY() * height));
|
||||
// Request canvas drawing.
|
||||
imageView.setHandsResult(handsResult);
|
||||
runOnUiThread(() -> imageView.update());
|
||||
});
|
||||
hands.setErrorListener(
|
||||
(message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
|
||||
|
||||
// ActivityResultLauncher to get an image from the gallery as Bitmap.
|
||||
ActivityResultLauncher<Intent> imageGetter =
|
||||
registerForActivityResult(
|
||||
new ActivityResultContracts.StartActivityForResult(),
|
||||
result -> {
|
||||
Intent resultIntent = result.getData();
|
||||
if (resultIntent != null && result.getResultCode() == RESULT_OK) {
|
||||
Bitmap bitmap = null;
|
||||
try {
|
||||
bitmap =
|
||||
MediaStore.Images.Media.getBitmap(
|
||||
this.getContentResolver(), resultIntent.getData());
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "Bitmap reading error:" + e);
|
||||
}
|
||||
if (bitmap != null) {
|
||||
hands.send(bitmap);
|
||||
}
|
||||
}
|
||||
});
|
||||
Intent gallery = new Intent(
|
||||
Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
|
||||
imageGetter.launch(gallery);
|
||||
```
|
||||
|
||||
#### Video Input
|
||||
|
||||
```java
|
||||
// For video input and result rendering with OpenGL.
|
||||
HandsOptions handsOptions =
|
||||
HandsOptions.builder()
|
||||
.setMode(HandsOptions.STREAMING_MODE) // API soon to become
|
||||
.setMaxNumHands(1) // setStaticImageMode(false)
|
||||
.setRunOnGpu(true).build();
|
||||
Hands hands = new Hands(this, handsOptions);
|
||||
hands.setErrorListener(
|
||||
(message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
|
||||
|
||||
// Initializes a new VideoInput instance and connects it to MediaPipe Hands.
|
||||
VideoInput videoInput = new VideoInput(this);
|
||||
videoInput.setNewFrameListener(
|
||||
textureFrame -> hands.send(textureFrame));
|
||||
|
||||
// Initializes a new GlSurfaceView with a ResultGlRenderer<HandsResult> instance
|
||||
// that provides the interfaces to run user-defined OpenGL rendering code.
|
||||
// See mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java
|
||||
// as an example.
|
||||
SolutionGlSurfaceView<HandsResult> glSurfaceView =
|
||||
new SolutionGlSurfaceView<>(
|
||||
this, hands.getGlContext(), hands.getGlMajorVersion());
|
||||
glSurfaceView.setSolutionResultRenderer(new HandsResultGlRenderer());
|
||||
glSurfaceView.setRenderInputImage(true);
|
||||
|
||||
hands.setResultListener(
|
||||
handsResult -> {
|
||||
NormalizedLandmark wristLandmark = Hands.getHandLandmark(
|
||||
handsResult, 0, HandLandmark.WRIST);
|
||||
Log.i(
|
||||
TAG,
|
||||
String.format(
|
||||
"MediaPipe Hand wrist normalized coordinates (value range: [0, 1]): x=%f, y=%f",
|
||||
wristLandmark.getX(), wristLandmark.getY()));
|
||||
// Request GL rendering.
|
||||
glSurfaceView.setRenderData(handsResult);
|
||||
glSurfaceView.requestRender();
|
||||
});
|
||||
|
||||
ActivityResultLauncher<Intent> videoGetter =
|
||||
registerForActivityResult(
|
||||
new ActivityResultContracts.StartActivityForResult(),
|
||||
result -> {
|
||||
Intent resultIntent = result.getData();
|
||||
if (resultIntent != null) {
|
||||
if (result.getResultCode() == RESULT_OK) {
|
||||
glSurfaceView.post(
|
||||
() ->
|
||||
videoInput.start(
|
||||
this,
|
||||
resultIntent.getData(),
|
||||
hands.getGlContext(),
|
||||
glSurfaceView.getWidth(),
|
||||
glSurfaceView.getHeight()));
|
||||
}
|
||||
}
|
||||
});
|
||||
Intent gallery =
|
||||
new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
|
||||
videoGetter.launch(gallery);
|
||||
```
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
|
||||
+23
-12
@@ -225,6 +225,7 @@ Supported configuration options:
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_drawing_styles = mp.solutions.drawing_styles
|
||||
mp_holistic = mp.solutions.holistic
|
||||
|
||||
# For static images:
|
||||
@@ -247,13 +248,18 @@ with mp_holistic.Holistic(
|
||||
# Draw pose, left and right hands, and face landmarks on the image.
|
||||
annotated_image = image.copy()
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
|
||||
annotated_image,
|
||||
results.face_landmarks,
|
||||
mp_holistic.FACEMESH_TESSELATION,
|
||||
landmark_drawing_spec=None,
|
||||
connection_drawing_spec=mp_drawing_styles
|
||||
.get_default_face_mesh_tesselation_style())
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
|
||||
annotated_image,
|
||||
results.pose_landmarks,
|
||||
mp_holistic.POSE_CONNECTIONS,
|
||||
landmark_drawing_spec=mp_drawing_styles.
|
||||
get_default_pose_landmarks_style())
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
# Plot pose world landmarks.
|
||||
mp_drawing.plot_landmarks(
|
||||
@@ -283,13 +289,18 @@ with mp_holistic.Holistic(
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
|
||||
image,
|
||||
results.face_landmarks,
|
||||
mp_holistic.FACEMESH_CONTOURS,
|
||||
landmark_drawing_spec=None,
|
||||
connection_drawing_spec=mp_drawing_styles
|
||||
.get_default_face_mesh_contours_style())
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
|
||||
image,
|
||||
results.pose_landmarks,
|
||||
mp_holistic.POSE_CONNECTIONS,
|
||||
landmark_drawing_spec=mp_drawing_styles
|
||||
.get_default_pose_landmarks_style())
|
||||
cv2.imshow('MediaPipe Holistic', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
|
||||
+107
-16
@@ -224,29 +224,33 @@ where object detection simply runs on every image. Default to `0.99`.
|
||||
|
||||
#### model_name
|
||||
|
||||
Name of the model to use for predicting 3D bounding box landmarks. Currently supports
|
||||
`{'Shoe', 'Chair', 'Cup', 'Camera'}`.
|
||||
Name of the model to use for predicting 3D bounding box landmarks. Currently
|
||||
supports `{'Shoe', 'Chair', 'Cup', 'Camera'}`. Default to `Shoe`.
|
||||
|
||||
#### focal_length
|
||||
|
||||
Camera focal length `(fx, fy)`, by default is defined in
|
||||
[NDC space](#ndc-space). To use focal length `(fx_pixel, fy_pixel)` in
|
||||
[pixel space](#pixel-space), users should provide `image_size` = `(image_width,
|
||||
image_height)` to enable conversions inside the API. For further details about
|
||||
NDC and pixel space, please see [Coordinate Systems](#coordinate-systems).
|
||||
By default, camera focal length defined in [NDC space](#ndc-space), i.e., `(fx,
|
||||
fy)`. Default to `(1.0, 1.0)`. To specify focal length in
|
||||
[pixel space](#pixel-space) instead, i.e., `(fx_pixel, fy_pixel)`, users should
|
||||
provide [`image_size`](#image_size) = `(image_width, image_height)` to enable
|
||||
conversions inside the API. For further details about NDC and pixel space,
|
||||
please see [Coordinate Systems](#coordinate-systems).
|
||||
|
||||
#### principal_point
|
||||
|
||||
Camera principal point `(px, py)`, by default is defined in
|
||||
[NDC space](#ndc-space). To use principal point `(px_pixel, py_pixel)` in
|
||||
[pixel space](#pixel-space), users should provide `image_size` = `(image_width,
|
||||
image_height)` to enable conversions inside the API. For further details about
|
||||
NDC and pixel space, please see [Coordinate Systems](#coordinate-systems).
|
||||
By default, camera principal point defined in [NDC space](#ndc-space), i.e.,
|
||||
`(px, py)`. Default to `(0.0, 0.0)`. To specify principal point in
|
||||
[pixel space](#pixel-space), i.e.,`(px_pixel, py_pixel)`, users should provide
|
||||
[`image_size`](#image_size) = `(image_width, image_height)` to enable
|
||||
conversions inside the API. For further details about NDC and pixel space,
|
||||
please see [Coordinate Systems](#coordinate-systems).
|
||||
|
||||
#### image_size
|
||||
|
||||
(**Optional**) size `(image_width, image_height)` of the input image, **ONLY**
|
||||
needed when use `focal_length` and `principal_point` in pixel space.
|
||||
**Specify only when [`focal_length`](#focal_length) and
|
||||
[`principal_point`](#principal_point) are specified in pixel space.**
|
||||
|
||||
Size of the input image, i.e., `(image_width, image_height)`.
|
||||
|
||||
### Output
|
||||
|
||||
@@ -356,6 +360,89 @@ with mp_objectron.Objectron(static_image_mode=False,
|
||||
cap.release()
|
||||
```
|
||||
|
||||
## JavaScript Solution API
|
||||
|
||||
Please first see general [introduction](../getting_started/javascript.md) on
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
|
||||
and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
* [staticImageMode](#static_image_mode)
|
||||
* [maxNumObjects](#max_num_objects)
|
||||
* [minDetectionConfidence](#min_detection_confidence)
|
||||
* [minTrackingConfidence](#min_tracking_confidence)
|
||||
* [modelName](#model_name)
|
||||
* [focalLength](#focal_length)
|
||||
* [principalPoint](#principal_point)
|
||||
* [imageSize](#image_size)
|
||||
|
||||
```html
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/control_utils_3d.js" crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/objectron/objectron.js" crossorigin="anonymous"></script>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="container">
|
||||
<video class="input_video"></video>
|
||||
<canvas class="output_canvas" width="1280px" height="720px"></canvas>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
```
|
||||
|
||||
```javascript
|
||||
<script type="module">
|
||||
const videoElement = document.getElementsByClassName('input_video')[0];
|
||||
const canvasElement = document.getElementsByClassName('output_canvas')[0];
|
||||
const canvasCtx = canvasElement.getContext('2d');
|
||||
|
||||
function onResults(results) {
|
||||
canvasCtx.save();
|
||||
canvasCtx.drawImage(
|
||||
results.image, 0, 0, canvasElement.width, canvasElement.height);
|
||||
if (!!results.objectDetections) {
|
||||
for (const detectedObject of results.objectDetections) {
|
||||
// Reformat keypoint information as landmarks, for easy drawing.
|
||||
const landmarks: mpObjectron.Point2D[] =
|
||||
detectedObject.keypoints.map(x => x.point2d);
|
||||
// Draw bounding box.
|
||||
drawingUtils.drawConnectors(canvasCtx, landmarks,
|
||||
mpObjectron.BOX_CONNECTIONS, {color: '#FF0000'});
|
||||
// Draw centroid.
|
||||
drawingUtils.drawLandmarks(canvasCtx, [landmarks[0]], {color: '#FFFFFF'});
|
||||
}
|
||||
}
|
||||
canvasCtx.restore();
|
||||
}
|
||||
|
||||
const objectron = new Objectron({locateFile: (file) => {
|
||||
return `https://cdn.jsdelivr.net/npm/@mediapipe/objectron/${file}`;
|
||||
}});
|
||||
objectron.setOptions({
|
||||
modelName: 'Chair',
|
||||
maxNumObjects: 3,
|
||||
});
|
||||
objectron.onResults(onResults);
|
||||
|
||||
const camera = new Camera(videoElement, {
|
||||
onFrame: async () => {
|
||||
await objectron.send({image: videoElement});
|
||||
},
|
||||
width: 1280,
|
||||
height: 720
|
||||
});
|
||||
camera.start();
|
||||
</script>
|
||||
```
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
@@ -561,11 +648,15 @@ py = -py_pixel * 2.0 / image_height + 1.0
|
||||
[Announcing the Objectron Dataset](https://ai.googleblog.com/2020/11/announcing-objectron-dataset.html)
|
||||
* Google AI Blog:
|
||||
[Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
|
||||
* Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in CVPR 2021
|
||||
* Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the
|
||||
Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in
|
||||
CVPR 2021
|
||||
* Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
|
||||
Shape Supervision](https://arxiv.org/abs/2003.03522)
|
||||
* Paper:
|
||||
[Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
|
||||
([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth Workshop on Computer Vision for AR/VR, CVPR 2020
|
||||
([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth
|
||||
Workshop on Computer Vision for AR/VR, CVPR 2020
|
||||
* [Models and model cards](./models.md#objectron)
|
||||
* [Web demo](https://code.mediapipe.dev/codepen/objectron)
|
||||
* [Python Colab](https://mediapipe.page.link/objectron_py_colab)
|
||||
|
||||
+79
-15
@@ -30,7 +30,8 @@ overlay of digital content and information on top of the physical world in
|
||||
augmented reality.
|
||||
|
||||
MediaPipe Pose is a ML solution for high-fidelity body pose tracking, inferring
|
||||
33 3D landmarks on the whole body from RGB video frames utilizing our
|
||||
33 3D landmarks and background segmentation mask on the whole body from RGB
|
||||
video frames utilizing our
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
research that also powers the
|
||||
[ML Kit Pose Detection API](https://developers.google.com/ml-kit/vision/pose-detection).
|
||||
@@ -49,11 +50,11 @@ The solution utilizes a two-step detector-tracker ML pipeline, proven to be
|
||||
effective in our [MediaPipe Hands](./hands.md) and
|
||||
[MediaPipe Face Mesh](./face_mesh.md) solutions. Using a detector, the pipeline
|
||||
first locates the person/pose region-of-interest (ROI) within the frame. The
|
||||
tracker subsequently predicts the pose landmarks within the ROI using the
|
||||
ROI-cropped frame as input. Note that for video use cases the detector is
|
||||
invoked only as needed, i.e., for the very first frame and when the tracker
|
||||
could no longer identify body pose presence in the previous frame. For other
|
||||
frames the pipeline simply derives the ROI from the previous frame’s pose
|
||||
tracker subsequently predicts the pose landmarks and segmentation mask within
|
||||
the ROI using the ROI-cropped frame as input. Note that for video use cases the
|
||||
detector is invoked only as needed, i.e., for the very first frame and when the
|
||||
tracker could no longer identify body pose presence in the previous frame. For
|
||||
other frames the pipeline simply derives the ROI from the previous frame’s pose
|
||||
landmarks.
|
||||
|
||||
The pipeline is implemented as a MediaPipe
|
||||
@@ -129,16 +130,19 @@ hip midpoints.
|
||||
The landmark model in MediaPipe Pose predicts the location of 33 pose landmarks
|
||||
(see figure below).
|
||||
|
||||
Please find more detail in the
|
||||
[BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html),
|
||||
this [paper](https://arxiv.org/abs/2006.10204) and
|
||||
[the model card](./models.md#pose), and the attributes in each landmark
|
||||
[below](#pose_landmarks).
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------------------------: |
|
||||
*Fig 4. 33 pose landmarks.* |
|
||||
|
||||
Optionally, MediaPipe Pose can predicts a full-body
|
||||
[segmentation mask](#segmentation_mask) represented as a two-class segmentation
|
||||
(human or background).
|
||||
|
||||
Please find more detail in the
|
||||
[BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html),
|
||||
this [paper](https://arxiv.org/abs/2006.10204),
|
||||
[the model card](./models.md#pose) and the [Output](#Output) section below.
|
||||
|
||||
## Solution APIs
|
||||
|
||||
### Cross-platform Configuration Options
|
||||
@@ -167,6 +171,18 @@ If set to `true`, the solution filters pose landmarks across different input
|
||||
images to reduce jitter, but ignored if [static_image_mode](#static_image_mode)
|
||||
is also set to `true`. Default to `true`.
|
||||
|
||||
#### enable_segmentation
|
||||
|
||||
If set to `true`, in addition to the pose landmarks the solution also generates
|
||||
the segmentation mask. Default to `false`.
|
||||
|
||||
#### smooth_segmentation
|
||||
|
||||
If set to `true`, the solution filters segmentation masks across different input
|
||||
images to reduce jitter. Ignored if [enable_segmentation](#enable_segmentation)
|
||||
is `false` or [static_image_mode](#static_image_mode) is `true`. Default to
|
||||
`true`.
|
||||
|
||||
#### min_detection_confidence
|
||||
|
||||
Minimum confidence value (`[0.0, 1.0]`) from the person-detection model for the
|
||||
@@ -211,6 +227,19 @@ the following:
|
||||
* `visibility`: Identical to that defined in the corresponding
|
||||
[pose_landmarks](#pose_landmarks).
|
||||
|
||||
#### segmentation_mask
|
||||
|
||||
The output segmentation mask, predicted only when
|
||||
[enable_segmentation](#enable_segmentation) is set to `true`. The mask has the
|
||||
same width and height as the input image, and contains values in `[0.0, 1.0]`
|
||||
where `1.0` and `0.0` indicate high certainty of a "human" and "background"
|
||||
pixel respectively. Please refer to the platform-specific usage examples below
|
||||
for usage details.
|
||||
|
||||
*Fig 6. Example of MediaPipe Pose segmentation mask.* |
|
||||
:-----------------------------------------------------------: |
|
||||
<video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/mobile/pose_segmentation.mp4" type="video/mp4"></video> |
|
||||
|
||||
### Python Solution API
|
||||
|
||||
Please first follow general [instructions](../getting_started/python.md) to
|
||||
@@ -222,6 +251,8 @@ Supported configuration options:
|
||||
* [static_image_mode](#static_image_mode)
|
||||
* [model_complexity](#model_complexity)
|
||||
* [smooth_landmarks](#smooth_landmarks)
|
||||
* [enable_segmentation](#enable_segmentation)
|
||||
* [smooth_segmentation](#smooth_segmentation)
|
||||
* [min_detection_confidence](#min_detection_confidence)
|
||||
* [min_tracking_confidence](#min_tracking_confidence)
|
||||
|
||||
@@ -229,13 +260,16 @@ Supported configuration options:
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_drawing_styles = mp.solutions.drawing_styles
|
||||
mp_pose = mp.solutions.pose
|
||||
|
||||
# For static images:
|
||||
IMAGE_FILES = []
|
||||
BG_COLOR = (192, 192, 192) # gray
|
||||
with mp_pose.Pose(
|
||||
static_image_mode=True,
|
||||
model_complexity=2,
|
||||
enable_segmentation=True,
|
||||
min_detection_confidence=0.5) as pose:
|
||||
for idx, file in enumerate(IMAGE_FILES):
|
||||
image = cv2.imread(file)
|
||||
@@ -250,10 +284,21 @@ with mp_pose.Pose(
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
|
||||
)
|
||||
# Draw pose landmarks on the image.
|
||||
|
||||
annotated_image = image.copy()
|
||||
# Draw segmentation on the image.
|
||||
# To improve segmentation around boundaries, consider applying a joint
|
||||
# bilateral filter to "results.segmentation_mask" with "image".
|
||||
condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1
|
||||
bg_image = np.zeros(image.shape, dtype=np.uint8)
|
||||
bg_image[:] = BG_COLOR
|
||||
annotated_image = np.where(condition, annotated_image, bg_image)
|
||||
# Draw pose landmarks on the image.
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
annotated_image,
|
||||
results.pose_landmarks,
|
||||
mp_pose.POSE_CONNECTIONS,
|
||||
landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
# Plot pose world landmarks.
|
||||
mp_drawing.plot_landmarks(
|
||||
@@ -283,7 +328,10 @@ with mp_pose.Pose(
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
image,
|
||||
results.pose_landmarks,
|
||||
mp_pose.POSE_CONNECTIONS,
|
||||
landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
|
||||
cv2.imshow('MediaPipe Pose', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
@@ -300,6 +348,8 @@ Supported configuration options:
|
||||
|
||||
* [modelComplexity](#model_complexity)
|
||||
* [smoothLandmarks](#smooth_landmarks)
|
||||
* [enableSegmentation](#enable_segmentation)
|
||||
* [smoothSegmentation](#smooth_segmentation)
|
||||
* [minDetectionConfidence](#min_detection_confidence)
|
||||
* [minTrackingConfidence](#min_tracking_confidence)
|
||||
|
||||
@@ -340,8 +390,20 @@ function onResults(results) {
|
||||
|
||||
canvasCtx.save();
|
||||
canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
|
||||
canvasCtx.drawImage(results.segmentationMask, 0, 0,
|
||||
canvasElement.width, canvasElement.height);
|
||||
|
||||
// Only overwrite existing pixels.
|
||||
canvasCtx.globalCompositeOperation = 'source-in';
|
||||
canvasCtx.fillStyle = '#00FF00';
|
||||
canvasCtx.fillRect(0, 0, canvasElement.width, canvasElement.height);
|
||||
|
||||
// Only overwrite missing pixels.
|
||||
canvasCtx.globalCompositeOperation = 'destination-atop';
|
||||
canvasCtx.drawImage(
|
||||
results.image, 0, 0, canvasElement.width, canvasElement.height);
|
||||
|
||||
canvasCtx.globalCompositeOperation = 'source-over';
|
||||
drawConnectors(canvasCtx, results.poseLandmarks, POSE_CONNECTIONS,
|
||||
{color: '#00FF00', lineWidth: 4});
|
||||
drawLandmarks(canvasCtx, results.poseLandmarks,
|
||||
@@ -357,6 +419,8 @@ const pose = new Pose({locateFile: (file) => {
|
||||
pose.setOptions({
|
||||
modelComplexity: 1,
|
||||
smoothLandmarks: true,
|
||||
enableSegmentation: true,
|
||||
smoothSegmentation: true,
|
||||
minDetectionConfidence: 0.5,
|
||||
minTrackingConfidence: 0.5
|
||||
});
|
||||
|
||||
@@ -96,6 +96,7 @@ Supported configuration options:
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_selfie_segmentation = mp.solutions.selfie_segmentation
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ has_toc: false
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
|
||||
@@ -140,6 +140,16 @@ mediapipe_proto_library(
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "graph_profile_calculator_proto",
|
||||
srcs = ["graph_profile_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "add_header_calculator",
|
||||
srcs = ["add_header_calculator.cc"],
|
||||
@@ -1200,3 +1210,45 @@ cc_test(
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "graph_profile_calculator",
|
||||
srcs = ["graph_profile_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":graph_profile_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_profile_cc_proto",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/api2:packet",
|
||||
"//mediapipe/framework/api2:port",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "graph_profile_calculator_test",
|
||||
srcs = ["graph_profile_calculator_test.cc"],
|
||||
deps = [
|
||||
":graph_profile_calculator",
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_profile_cc_proto",
|
||||
"//mediapipe/framework:test_calculators",
|
||||
"//mediapipe/framework/deps:clock",
|
||||
"//mediapipe/framework/deps:message_matchers",
|
||||
"//mediapipe/framework/port:core_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:threadpool",
|
||||
"//mediapipe/framework/tool:simulation_clock_executor",
|
||||
"//mediapipe/framework/tool:sink",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_absl//absl/time",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -24,6 +24,9 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kDataTag[] = "DATA";
|
||||
constexpr char kHeaderTag[] = "HEADER";
|
||||
|
||||
class AddHeaderCalculatorTest : public ::testing::Test {};
|
||||
|
||||
TEST_F(AddHeaderCalculatorTest, HeaderStream) {
|
||||
@@ -36,11 +39,11 @@ TEST_F(AddHeaderCalculatorTest, HeaderStream) {
|
||||
CalculatorRunner runner(node);
|
||||
|
||||
// Set header and add 5 packets.
|
||||
runner.MutableInputs()->Tag("HEADER").header =
|
||||
runner.MutableInputs()->Tag(kHeaderTag).header =
|
||||
Adopt(new std::string("my_header"));
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
|
||||
runner.MutableInputs()->Tag(kDataTag).packets.push_back(packet);
|
||||
}
|
||||
|
||||
// Run calculator.
|
||||
@@ -85,13 +88,14 @@ TEST_F(AddHeaderCalculatorTest, NoPacketsOnHeaderStream) {
|
||||
CalculatorRunner runner(node);
|
||||
|
||||
// Set header and add 5 packets.
|
||||
runner.MutableInputs()->Tag("HEADER").header =
|
||||
runner.MutableInputs()->Tag(kHeaderTag).header =
|
||||
Adopt(new std::string("my_header"));
|
||||
runner.MutableInputs()->Tag("HEADER").packets.push_back(
|
||||
Adopt(new std::string("not allowed")));
|
||||
runner.MutableInputs()
|
||||
->Tag(kHeaderTag)
|
||||
.packets.push_back(Adopt(new std::string("not allowed")));
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
|
||||
runner.MutableInputs()->Tag(kDataTag).packets.push_back(packet);
|
||||
}
|
||||
|
||||
// Run calculator.
|
||||
@@ -108,11 +112,11 @@ TEST_F(AddHeaderCalculatorTest, InputSidePacket) {
|
||||
CalculatorRunner runner(node);
|
||||
|
||||
// Set header and add 5 packets.
|
||||
runner.MutableSidePackets()->Tag("HEADER") =
|
||||
runner.MutableSidePackets()->Tag(kHeaderTag) =
|
||||
Adopt(new std::string("my_header"));
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
|
||||
runner.MutableInputs()->Tag(kDataTag).packets.push_back(packet);
|
||||
}
|
||||
|
||||
// Run calculator.
|
||||
@@ -143,13 +147,13 @@ TEST_F(AddHeaderCalculatorTest, UsingBothSideInputAndStream) {
|
||||
CalculatorRunner runner(node);
|
||||
|
||||
// Set both headers and add 5 packets.
|
||||
runner.MutableSidePackets()->Tag("HEADER") =
|
||||
runner.MutableSidePackets()->Tag(kHeaderTag) =
|
||||
Adopt(new std::string("my_header"));
|
||||
runner.MutableSidePackets()->Tag("HEADER") =
|
||||
runner.MutableSidePackets()->Tag(kHeaderTag) =
|
||||
Adopt(new std::string("my_header"));
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
|
||||
runner.MutableInputs()->Tag(kDataTag).packets.push_back(packet);
|
||||
}
|
||||
|
||||
// Run should fail because header can only be provided one way.
|
||||
|
||||
@@ -19,6 +19,13 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kIncrementTag[] = "INCREMENT";
|
||||
constexpr char kInitialValueTag[] = "INITIAL_VALUE";
|
||||
constexpr char kBatchSizeTag[] = "BATCH_SIZE";
|
||||
constexpr char kErrorCountTag[] = "ERROR_COUNT";
|
||||
constexpr char kMaxCountTag[] = "MAX_COUNT";
|
||||
constexpr char kErrorOnOpenTag[] = "ERROR_ON_OPEN";
|
||||
|
||||
// Source calculator that produces MAX_COUNT*BATCH_SIZE int packets of
|
||||
// sequential numbers from INITIAL_VALUE (default 0) with a common
|
||||
// difference of INCREMENT (default 1) between successive numbers (with
|
||||
@@ -33,53 +40,53 @@ class CountingSourceCalculator : public CalculatorBase {
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Outputs().Index(0).Set<int>();
|
||||
|
||||
if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN")) {
|
||||
cc->InputSidePackets().Tag("ERROR_ON_OPEN").Set<bool>();
|
||||
if (cc->InputSidePackets().HasTag(kErrorOnOpenTag)) {
|
||||
cc->InputSidePackets().Tag(kErrorOnOpenTag).Set<bool>();
|
||||
}
|
||||
|
||||
RET_CHECK(cc->InputSidePackets().HasTag("MAX_COUNT") ||
|
||||
cc->InputSidePackets().HasTag("ERROR_COUNT"));
|
||||
if (cc->InputSidePackets().HasTag("MAX_COUNT")) {
|
||||
cc->InputSidePackets().Tag("MAX_COUNT").Set<int>();
|
||||
RET_CHECK(cc->InputSidePackets().HasTag(kMaxCountTag) ||
|
||||
cc->InputSidePackets().HasTag(kErrorCountTag));
|
||||
if (cc->InputSidePackets().HasTag(kMaxCountTag)) {
|
||||
cc->InputSidePackets().Tag(kMaxCountTag).Set<int>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("ERROR_COUNT")) {
|
||||
cc->InputSidePackets().Tag("ERROR_COUNT").Set<int>();
|
||||
if (cc->InputSidePackets().HasTag(kErrorCountTag)) {
|
||||
cc->InputSidePackets().Tag(kErrorCountTag).Set<int>();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("BATCH_SIZE")) {
|
||||
cc->InputSidePackets().Tag("BATCH_SIZE").Set<int>();
|
||||
if (cc->InputSidePackets().HasTag(kBatchSizeTag)) {
|
||||
cc->InputSidePackets().Tag(kBatchSizeTag).Set<int>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("INITIAL_VALUE")) {
|
||||
cc->InputSidePackets().Tag("INITIAL_VALUE").Set<int>();
|
||||
if (cc->InputSidePackets().HasTag(kInitialValueTag)) {
|
||||
cc->InputSidePackets().Tag(kInitialValueTag).Set<int>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("INCREMENT")) {
|
||||
cc->InputSidePackets().Tag("INCREMENT").Set<int>();
|
||||
if (cc->InputSidePackets().HasTag(kIncrementTag)) {
|
||||
cc->InputSidePackets().Tag(kIncrementTag).Set<int>();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override {
|
||||
if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN") &&
|
||||
cc->InputSidePackets().Tag("ERROR_ON_OPEN").Get<bool>()) {
|
||||
if (cc->InputSidePackets().HasTag(kErrorOnOpenTag) &&
|
||||
cc->InputSidePackets().Tag(kErrorOnOpenTag).Get<bool>()) {
|
||||
return absl::NotFoundError("expected error");
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("ERROR_COUNT")) {
|
||||
error_count_ = cc->InputSidePackets().Tag("ERROR_COUNT").Get<int>();
|
||||
if (cc->InputSidePackets().HasTag(kErrorCountTag)) {
|
||||
error_count_ = cc->InputSidePackets().Tag(kErrorCountTag).Get<int>();
|
||||
RET_CHECK_LE(0, error_count_);
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("MAX_COUNT")) {
|
||||
max_count_ = cc->InputSidePackets().Tag("MAX_COUNT").Get<int>();
|
||||
if (cc->InputSidePackets().HasTag(kMaxCountTag)) {
|
||||
max_count_ = cc->InputSidePackets().Tag(kMaxCountTag).Get<int>();
|
||||
RET_CHECK_LE(0, max_count_);
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("BATCH_SIZE")) {
|
||||
batch_size_ = cc->InputSidePackets().Tag("BATCH_SIZE").Get<int>();
|
||||
if (cc->InputSidePackets().HasTag(kBatchSizeTag)) {
|
||||
batch_size_ = cc->InputSidePackets().Tag(kBatchSizeTag).Get<int>();
|
||||
RET_CHECK_LT(0, batch_size_);
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("INITIAL_VALUE")) {
|
||||
counter_ = cc->InputSidePackets().Tag("INITIAL_VALUE").Get<int>();
|
||||
if (cc->InputSidePackets().HasTag(kInitialValueTag)) {
|
||||
counter_ = cc->InputSidePackets().Tag(kInitialValueTag).Get<int>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("INCREMENT")) {
|
||||
increment_ = cc->InputSidePackets().Tag("INCREMENT").Get<int>();
|
||||
if (cc->InputSidePackets().HasTag(kIncrementTag)) {
|
||||
increment_ = cc->InputSidePackets().Tag(kIncrementTag).Get<int>();
|
||||
RET_CHECK_LT(0, increment_);
|
||||
}
|
||||
RET_CHECK(error_count_ >= 0 || max_count_ >= 0);
|
||||
|
||||
@@ -35,11 +35,14 @@
|
||||
// }
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kFloatVectorTag[] = "FLOAT_VECTOR";
|
||||
constexpr char kEncodedTag[] = "ENCODED";
|
||||
|
||||
class DequantizeByteArrayCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag("ENCODED").Set<std::string>();
|
||||
cc->Outputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
|
||||
cc->Inputs().Tag(kEncodedTag).Set<std::string>();
|
||||
cc->Outputs().Tag(kFloatVectorTag).Set<std::vector<float>>();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -66,7 +69,7 @@ class DequantizeByteArrayCalculator : public CalculatorBase {
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
const std::string& encoded =
|
||||
cc->Inputs().Tag("ENCODED").Value().Get<std::string>();
|
||||
cc->Inputs().Tag(kEncodedTag).Value().Get<std::string>();
|
||||
std::vector<float> float_vector;
|
||||
float_vector.reserve(encoded.length());
|
||||
for (int i = 0; i < encoded.length(); ++i) {
|
||||
@@ -74,7 +77,7 @@ class DequantizeByteArrayCalculator : public CalculatorBase {
|
||||
static_cast<unsigned char>(encoded.at(i)) * scalar_ + bias_);
|
||||
}
|
||||
cc->Outputs()
|
||||
.Tag("FLOAT_VECTOR")
|
||||
.Tag(kFloatVectorTag)
|
||||
.AddPacket(MakePacket<std::vector<float>>(float_vector)
|
||||
.At(cc->InputTimestamp()));
|
||||
return absl::OkStatus();
|
||||
|
||||
@@ -25,6 +25,9 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kFloatVectorTag[] = "FLOAT_VECTOR";
|
||||
constexpr char kEncodedTag[] = "ENCODED";
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
@@ -39,8 +42,10 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
MakePacket<std::string>(empty_string).At(Timestamp(0)));
|
||||
runner.MutableInputs()
|
||||
->Tag(kEncodedTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::string>(empty_string).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
@@ -64,8 +69,10 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
MakePacket<std::string>(empty_string).At(Timestamp(0)));
|
||||
runner.MutableInputs()
|
||||
->Tag(kEncodedTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::string>(empty_string).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
@@ -89,8 +96,10 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
MakePacket<std::string>(empty_string).At(Timestamp(0)));
|
||||
runner.MutableInputs()
|
||||
->Tag(kEncodedTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::string>(empty_string).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
@@ -114,14 +123,16 @@ TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
MakePacket<std::string>(
|
||||
std::string(reinterpret_cast<char const*>(input), 4))
|
||||
.At(Timestamp(0)));
|
||||
runner.MutableInputs()
|
||||
->Tag(kEncodedTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::string>(
|
||||
std::string(reinterpret_cast<char const*>(input), 4))
|
||||
.At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& outputs =
|
||||
runner.Outputs().Tag("FLOAT_VECTOR").packets;
|
||||
runner.Outputs().Tag(kFloatVectorTag).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
const std::vector<float>& result = outputs[0].Get<std::vector<float>>();
|
||||
ASSERT_FALSE(result.empty());
|
||||
|
||||
@@ -24,6 +24,11 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kFinishedTag[] = "FINISHED";
|
||||
constexpr char kAllowTag[] = "ALLOW";
|
||||
constexpr char kMaxInFlightTag[] = "MAX_IN_FLIGHT";
|
||||
constexpr char kOptionsTag[] = "OPTIONS";
|
||||
|
||||
// FlowLimiterCalculator is used to limit the number of frames in flight
|
||||
// by dropping input frames when necessary.
|
||||
//
|
||||
@@ -69,16 +74,19 @@ class FlowLimiterCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
auto& side_inputs = cc->InputSidePackets();
|
||||
side_inputs.Tag("OPTIONS").Set<FlowLimiterCalculatorOptions>().Optional();
|
||||
cc->Inputs().Tag("OPTIONS").Set<FlowLimiterCalculatorOptions>().Optional();
|
||||
side_inputs.Tag(kOptionsTag).Set<FlowLimiterCalculatorOptions>().Optional();
|
||||
cc->Inputs()
|
||||
.Tag(kOptionsTag)
|
||||
.Set<FlowLimiterCalculatorOptions>()
|
||||
.Optional();
|
||||
RET_CHECK_GE(cc->Inputs().NumEntries(""), 1);
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(""); ++i) {
|
||||
cc->Inputs().Get("", i).SetAny();
|
||||
cc->Outputs().Get("", i).SetSameAs(&(cc->Inputs().Get("", i)));
|
||||
}
|
||||
cc->Inputs().Get("FINISHED", 0).SetAny();
|
||||
cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Set<int>().Optional();
|
||||
cc->Outputs().Tag("ALLOW").Set<bool>().Optional();
|
||||
cc->InputSidePackets().Tag(kMaxInFlightTag).Set<int>().Optional();
|
||||
cc->Outputs().Tag(kAllowTag).Set<bool>().Optional();
|
||||
cc->SetInputStreamHandler("ImmediateInputStreamHandler");
|
||||
cc->SetProcessTimestampBounds(true);
|
||||
return absl::OkStatus();
|
||||
@@ -87,9 +95,9 @@ class FlowLimiterCalculator : public CalculatorBase {
|
||||
absl::Status Open(CalculatorContext* cc) final {
|
||||
options_ = cc->Options<FlowLimiterCalculatorOptions>();
|
||||
options_ = tool::RetrieveOptions(options_, cc->InputSidePackets());
|
||||
if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
|
||||
if (cc->InputSidePackets().HasTag(kMaxInFlightTag)) {
|
||||
options_.set_max_in_flight(
|
||||
cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Get<int>());
|
||||
cc->InputSidePackets().Tag(kMaxInFlightTag).Get<int>());
|
||||
}
|
||||
input_queues_.resize(cc->Inputs().NumEntries(""));
|
||||
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs())));
|
||||
@@ -104,8 +112,8 @@ class FlowLimiterCalculator : public CalculatorBase {
|
||||
|
||||
// Outputs a packet indicating whether a frame was sent or dropped.
|
||||
void SendAllow(bool allow, Timestamp ts, CalculatorContext* cc) {
|
||||
if (cc->Outputs().HasTag("ALLOW")) {
|
||||
cc->Outputs().Tag("ALLOW").AddPacket(MakePacket<bool>(allow).At(ts));
|
||||
if (cc->Outputs().HasTag(kAllowTag)) {
|
||||
cc->Outputs().Tag(kAllowTag).AddPacket(MakePacket<bool>(allow).At(ts));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,7 +163,7 @@ class FlowLimiterCalculator : public CalculatorBase {
|
||||
options_ = tool::RetrieveOptions(options_, cc->Inputs());
|
||||
|
||||
// Process the FINISHED input stream.
|
||||
Packet finished_packet = cc->Inputs().Tag("FINISHED").Value();
|
||||
Packet finished_packet = cc->Inputs().Tag(kFinishedTag).Value();
|
||||
if (finished_packet.Timestamp() == cc->InputTimestamp()) {
|
||||
while (!frames_in_flight_.empty() &&
|
||||
frames_in_flight_.front() <= finished_packet.Timestamp()) {
|
||||
@@ -210,8 +218,8 @@ class FlowLimiterCalculator : public CalculatorBase {
|
||||
Timestamp bound =
|
||||
cc->Inputs().Get("", 0).Value().Timestamp().NextAllowedInStream();
|
||||
SetNextTimestampBound(bound, &cc->Outputs().Get("", 0));
|
||||
if (cc->Outputs().HasTag("ALLOW")) {
|
||||
SetNextTimestampBound(bound, &cc->Outputs().Tag("ALLOW"));
|
||||
if (cc->Outputs().HasTag(kAllowTag)) {
|
||||
SetNextTimestampBound(bound, &cc->Outputs().Tag(kAllowTag));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,13 @@
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kDropTimestampsTag[] = "DROP_TIMESTAMPS";
|
||||
constexpr char kClockTag[] = "CLOCK";
|
||||
constexpr char kWarmupTimeTag[] = "WARMUP_TIME";
|
||||
constexpr char kSleepTimeTag[] = "SLEEP_TIME";
|
||||
constexpr char kPacketTag[] = "PACKET";
|
||||
|
||||
// A simple Semaphore for synchronizing test threads.
|
||||
class AtomicSemaphore {
|
||||
public:
|
||||
@@ -204,17 +211,17 @@ TEST_F(FlowLimiterCalculatorSemaphoreTest, FramesDropped) {
|
||||
class SleepCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag("PACKET").SetAny();
|
||||
cc->Outputs().Tag("PACKET").SetSameAs(&cc->Inputs().Tag("PACKET"));
|
||||
cc->InputSidePackets().Tag("SLEEP_TIME").Set<int64>();
|
||||
cc->InputSidePackets().Tag("WARMUP_TIME").Set<int64>();
|
||||
cc->InputSidePackets().Tag("CLOCK").Set<mediapipe::Clock*>();
|
||||
cc->Inputs().Tag(kPacketTag).SetAny();
|
||||
cc->Outputs().Tag(kPacketTag).SetSameAs(&cc->Inputs().Tag(kPacketTag));
|
||||
cc->InputSidePackets().Tag(kSleepTimeTag).Set<int64>();
|
||||
cc->InputSidePackets().Tag(kWarmupTimeTag).Set<int64>();
|
||||
cc->InputSidePackets().Tag(kClockTag).Set<mediapipe::Clock*>();
|
||||
cc->SetTimestampOffset(0);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) final {
|
||||
clock_ = cc->InputSidePackets().Tag("CLOCK").Get<mediapipe::Clock*>();
|
||||
clock_ = cc->InputSidePackets().Tag(kClockTag).Get<mediapipe::Clock*>();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -222,10 +229,12 @@ class SleepCalculator : public CalculatorBase {
|
||||
++packet_count;
|
||||
absl::Duration sleep_time = absl::Microseconds(
|
||||
packet_count == 1
|
||||
? cc->InputSidePackets().Tag("WARMUP_TIME").Get<int64>()
|
||||
: cc->InputSidePackets().Tag("SLEEP_TIME").Get<int64>());
|
||||
? cc->InputSidePackets().Tag(kWarmupTimeTag).Get<int64>()
|
||||
: cc->InputSidePackets().Tag(kSleepTimeTag).Get<int64>());
|
||||
clock_->Sleep(sleep_time);
|
||||
cc->Outputs().Tag("PACKET").AddPacket(cc->Inputs().Tag("PACKET").Value());
|
||||
cc->Outputs()
|
||||
.Tag(kPacketTag)
|
||||
.AddPacket(cc->Inputs().Tag(kPacketTag).Value());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -240,24 +249,27 @@ REGISTER_CALCULATOR(SleepCalculator);
|
||||
class DropCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag("PACKET").SetAny();
|
||||
cc->Outputs().Tag("PACKET").SetSameAs(&cc->Inputs().Tag("PACKET"));
|
||||
cc->InputSidePackets().Tag("DROP_TIMESTAMPS").Set<bool>();
|
||||
cc->Inputs().Tag(kPacketTag).SetAny();
|
||||
cc->Outputs().Tag(kPacketTag).SetSameAs(&cc->Inputs().Tag(kPacketTag));
|
||||
cc->InputSidePackets().Tag(kDropTimestampsTag).Set<bool>();
|
||||
cc->SetProcessTimestampBounds(true);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
if (!cc->Inputs().Tag("PACKET").Value().IsEmpty()) {
|
||||
if (!cc->Inputs().Tag(kPacketTag).Value().IsEmpty()) {
|
||||
++packet_count;
|
||||
}
|
||||
bool drop = (packet_count == 3);
|
||||
if (!drop && !cc->Inputs().Tag("PACKET").Value().IsEmpty()) {
|
||||
cc->Outputs().Tag("PACKET").AddPacket(cc->Inputs().Tag("PACKET").Value());
|
||||
if (!drop && !cc->Inputs().Tag(kPacketTag).Value().IsEmpty()) {
|
||||
cc->Outputs()
|
||||
.Tag(kPacketTag)
|
||||
.AddPacket(cc->Inputs().Tag(kPacketTag).Value());
|
||||
}
|
||||
if (!drop || !cc->InputSidePackets().Tag("DROP_TIMESTAMPS").Get<bool>()) {
|
||||
cc->Outputs().Tag("PACKET").SetNextTimestampBound(
|
||||
cc->InputTimestamp().NextAllowedInStream());
|
||||
if (!drop || !cc->InputSidePackets().Tag(kDropTimestampsTag).Get<bool>()) {
|
||||
cc->Outputs()
|
||||
.Tag(kPacketTag)
|
||||
.SetNextTimestampBound(cc->InputTimestamp().NextAllowedInStream());
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -21,6 +21,11 @@
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kStateChangeTag[] = "STATE_CHANGE";
|
||||
constexpr char kDisallowTag[] = "DISALLOW";
|
||||
constexpr char kAllowTag[] = "ALLOW";
|
||||
|
||||
enum GateState {
|
||||
GATE_UNINITIALIZED,
|
||||
GATE_ALLOW,
|
||||
@@ -59,8 +64,9 @@ std::string ToString(GateState state) {
|
||||
// ALLOW or DISALLOW can also be specified as an input side packet. The rules
|
||||
// for evaluation remain the same as above.
|
||||
//
|
||||
// ALLOW/DISALLOW inputs must be specified either using input stream or
|
||||
// via input side packet but not both.
|
||||
// ALLOW/DISALLOW inputs must be specified either using input stream or via
|
||||
// input side packet but not both. If neither is specified, the behavior is then
|
||||
// determined by the "allow" field in the calculator options.
|
||||
//
|
||||
// Intended to be used with the default input stream handler, which synchronizes
|
||||
// all data input streams with the ALLOW/DISALLOW control input stream.
|
||||
@@ -83,30 +89,33 @@ class GateCalculator : public CalculatorBase {
|
||||
GateCalculator() {}
|
||||
|
||||
static absl::Status CheckAndInitAllowDisallowInputs(CalculatorContract* cc) {
|
||||
bool input_via_side_packet = cc->InputSidePackets().HasTag("ALLOW") ||
|
||||
cc->InputSidePackets().HasTag("DISALLOW");
|
||||
bool input_via_side_packet = cc->InputSidePackets().HasTag(kAllowTag) ||
|
||||
cc->InputSidePackets().HasTag(kDisallowTag);
|
||||
bool input_via_stream =
|
||||
cc->Inputs().HasTag("ALLOW") || cc->Inputs().HasTag("DISALLOW");
|
||||
// Only one of input_side_packet or input_stream may specify ALLOW/DISALLOW
|
||||
// input.
|
||||
RET_CHECK(input_via_side_packet ^ input_via_stream);
|
||||
cc->Inputs().HasTag(kAllowTag) || cc->Inputs().HasTag(kDisallowTag);
|
||||
|
||||
// Only one of input_side_packet or input_stream may specify
|
||||
// ALLOW/DISALLOW input.
|
||||
if (input_via_side_packet) {
|
||||
RET_CHECK(cc->InputSidePackets().HasTag("ALLOW") ^
|
||||
cc->InputSidePackets().HasTag("DISALLOW"));
|
||||
RET_CHECK(!input_via_stream);
|
||||
RET_CHECK(cc->InputSidePackets().HasTag(kAllowTag) ^
|
||||
cc->InputSidePackets().HasTag(kDisallowTag));
|
||||
|
||||
if (cc->InputSidePackets().HasTag("ALLOW")) {
|
||||
cc->InputSidePackets().Tag("ALLOW").Set<bool>();
|
||||
if (cc->InputSidePackets().HasTag(kAllowTag)) {
|
||||
cc->InputSidePackets().Tag(kAllowTag).Set<bool>().Optional();
|
||||
} else {
|
||||
cc->InputSidePackets().Tag("DISALLOW").Set<bool>();
|
||||
cc->InputSidePackets().Tag(kDisallowTag).Set<bool>().Optional();
|
||||
}
|
||||
} else {
|
||||
RET_CHECK(cc->Inputs().HasTag("ALLOW") ^ cc->Inputs().HasTag("DISALLOW"));
|
||||
}
|
||||
if (input_via_stream) {
|
||||
RET_CHECK(!input_via_side_packet);
|
||||
RET_CHECK(cc->Inputs().HasTag(kAllowTag) ^
|
||||
cc->Inputs().HasTag(kDisallowTag));
|
||||
|
||||
if (cc->Inputs().HasTag("ALLOW")) {
|
||||
cc->Inputs().Tag("ALLOW").Set<bool>();
|
||||
if (cc->Inputs().HasTag(kAllowTag)) {
|
||||
cc->Inputs().Tag(kAllowTag).Set<bool>();
|
||||
} else {
|
||||
cc->Inputs().Tag("DISALLOW").Set<bool>();
|
||||
cc->Inputs().Tag(kDisallowTag).Set<bool>();
|
||||
}
|
||||
}
|
||||
return absl::OkStatus();
|
||||
@@ -125,23 +134,22 @@ class GateCalculator : public CalculatorBase {
|
||||
cc->Outputs().Get("", i).SetSameAs(&cc->Inputs().Get("", i));
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag("STATE_CHANGE")) {
|
||||
cc->Outputs().Tag("STATE_CHANGE").Set<bool>();
|
||||
if (cc->Outputs().HasTag(kStateChangeTag)) {
|
||||
cc->Outputs().Tag(kStateChangeTag).Set<bool>();
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) final {
|
||||
use_side_packet_for_allow_disallow_ = false;
|
||||
if (cc->InputSidePackets().HasTag("ALLOW")) {
|
||||
if (cc->InputSidePackets().HasTag(kAllowTag)) {
|
||||
use_side_packet_for_allow_disallow_ = true;
|
||||
allow_by_side_packet_decision_ =
|
||||
cc->InputSidePackets().Tag("ALLOW").Get<bool>();
|
||||
} else if (cc->InputSidePackets().HasTag("DISALLOW")) {
|
||||
cc->InputSidePackets().Tag(kAllowTag).Get<bool>();
|
||||
} else if (cc->InputSidePackets().HasTag(kDisallowTag)) {
|
||||
use_side_packet_for_allow_disallow_ = true;
|
||||
allow_by_side_packet_decision_ =
|
||||
!cc->InputSidePackets().Tag("DISALLOW").Get<bool>();
|
||||
!cc->InputSidePackets().Tag(kDisallowTag).Get<bool>();
|
||||
}
|
||||
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
@@ -152,26 +160,34 @@ class GateCalculator : public CalculatorBase {
|
||||
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
|
||||
empty_packets_as_allow_ = options.empty_packets_as_allow();
|
||||
|
||||
if (!use_side_packet_for_allow_disallow_ &&
|
||||
!cc->Inputs().HasTag(kAllowTag) && !cc->Inputs().HasTag(kDisallowTag)) {
|
||||
use_option_for_allow_disallow_ = true;
|
||||
allow_by_option_decision_ = options.allow();
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
bool allow = empty_packets_as_allow_;
|
||||
if (use_side_packet_for_allow_disallow_) {
|
||||
if (use_option_for_allow_disallow_) {
|
||||
allow = allow_by_option_decision_;
|
||||
} else if (use_side_packet_for_allow_disallow_) {
|
||||
allow = allow_by_side_packet_decision_;
|
||||
} else {
|
||||
if (cc->Inputs().HasTag("ALLOW") &&
|
||||
!cc->Inputs().Tag("ALLOW").IsEmpty()) {
|
||||
allow = cc->Inputs().Tag("ALLOW").Get<bool>();
|
||||
if (cc->Inputs().HasTag(kAllowTag) &&
|
||||
!cc->Inputs().Tag(kAllowTag).IsEmpty()) {
|
||||
allow = cc->Inputs().Tag(kAllowTag).Get<bool>();
|
||||
}
|
||||
if (cc->Inputs().HasTag("DISALLOW") &&
|
||||
!cc->Inputs().Tag("DISALLOW").IsEmpty()) {
|
||||
allow = !cc->Inputs().Tag("DISALLOW").Get<bool>();
|
||||
if (cc->Inputs().HasTag(kDisallowTag) &&
|
||||
!cc->Inputs().Tag(kDisallowTag).IsEmpty()) {
|
||||
allow = !cc->Inputs().Tag(kDisallowTag).Get<bool>();
|
||||
}
|
||||
}
|
||||
const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
|
||||
|
||||
if (cc->Outputs().HasTag("STATE_CHANGE")) {
|
||||
if (cc->Outputs().HasTag(kStateChangeTag)) {
|
||||
if (last_gate_state_ != GATE_UNINITIALIZED &&
|
||||
last_gate_state_ != new_gate_state) {
|
||||
VLOG(2) << "State transition in " << cc->NodeName() << " @ "
|
||||
@@ -179,7 +195,7 @@ class GateCalculator : public CalculatorBase {
|
||||
<< ToString(last_gate_state_) << " to "
|
||||
<< ToString(new_gate_state);
|
||||
cc->Outputs()
|
||||
.Tag("STATE_CHANGE")
|
||||
.Tag(kStateChangeTag)
|
||||
.AddPacket(MakePacket<bool>(allow).At(cc->InputTimestamp()));
|
||||
}
|
||||
}
|
||||
@@ -211,8 +227,10 @@ class GateCalculator : public CalculatorBase {
|
||||
GateState last_gate_state_ = GATE_UNINITIALIZED;
|
||||
int num_data_streams_;
|
||||
bool empty_packets_as_allow_;
|
||||
bool use_side_packet_for_allow_disallow_;
|
||||
bool use_side_packet_for_allow_disallow_ = false;
|
||||
bool allow_by_side_packet_decision_;
|
||||
bool use_option_for_allow_disallow_ = false;
|
||||
bool allow_by_option_decision_;
|
||||
};
|
||||
REGISTER_CALCULATOR(GateCalculator);
|
||||
|
||||
|
||||
@@ -29,4 +29,8 @@ message GateCalculatorOptions {
|
||||
// disallowing the corresponding packets in the data input streams. Setting
|
||||
// this option to true inverts that, allowing the data packets to go through.
|
||||
optional bool empty_packets_as_allow = 1;
|
||||
|
||||
// Whether to allow or disallow the input streams to pass when no
|
||||
// ALLOW/DISALLOW input or side input is specified.
|
||||
optional bool allow = 2 [default = false];
|
||||
}
|
||||
|
||||
@@ -22,6 +22,9 @@ namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kDisallowTag[] = "DISALLOW";
|
||||
constexpr char kAllowTag[] = "ALLOW";
|
||||
|
||||
class GateCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
// Helper to run a graph and return status.
|
||||
@@ -110,6 +113,68 @@ TEST_F(GateCalculatorTest, InvalidInputs) {
|
||||
)")));
|
||||
}
|
||||
|
||||
TEST_F(GateCalculatorTest, AllowByALLOWOptionToTrue) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
output_stream: "test_output"
|
||||
options: {
|
||||
[mediapipe.GateCalculatorOptions.ext] {
|
||||
allow: true
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, true);
|
||||
constexpr int64 kTimestampValue1 = 43;
|
||||
RunTimeStep(kTimestampValue1, false);
|
||||
|
||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||
ASSERT_EQ(2, output.size());
|
||||
EXPECT_EQ(kTimestampValue0, output[0].Timestamp().Value());
|
||||
EXPECT_EQ(kTimestampValue1, output[1].Timestamp().Value());
|
||||
EXPECT_EQ(true, output[0].Get<bool>());
|
||||
EXPECT_EQ(false, output[1].Get<bool>());
|
||||
}
|
||||
|
||||
TEST_F(GateCalculatorTest, DisallowByALLOWOptionSetToFalse) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
output_stream: "test_output"
|
||||
options: {
|
||||
[mediapipe.GateCalculatorOptions.ext] {
|
||||
allow: false
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, true);
|
||||
constexpr int64 kTimestampValue1 = 43;
|
||||
RunTimeStep(kTimestampValue1, false);
|
||||
|
||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||
ASSERT_EQ(0, output.size());
|
||||
}
|
||||
|
||||
TEST_F(GateCalculatorTest, DisallowByALLOWOptionNotSet) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
output_stream: "test_output"
|
||||
)");
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, true);
|
||||
constexpr int64 kTimestampValue1 = 43;
|
||||
RunTimeStep(kTimestampValue1, false);
|
||||
|
||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||
ASSERT_EQ(0, output.size());
|
||||
}
|
||||
|
||||
TEST_F(GateCalculatorTest, AllowByALLOWSidePacketSetToTrue) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
@@ -117,7 +182,7 @@ TEST_F(GateCalculatorTest, AllowByALLOWSidePacketSetToTrue) {
|
||||
input_stream: "test_input"
|
||||
output_stream: "test_output"
|
||||
)");
|
||||
runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(true));
|
||||
runner()->MutableSidePackets()->Tag(kAllowTag) = Adopt(new bool(true));
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, true);
|
||||
@@ -139,7 +204,7 @@ TEST_F(GateCalculatorTest, AllowByDisallowSidePacketSetToFalse) {
|
||||
input_stream: "test_input"
|
||||
output_stream: "test_output"
|
||||
)");
|
||||
runner()->MutableSidePackets()->Tag("DISALLOW") = Adopt(new bool(false));
|
||||
runner()->MutableSidePackets()->Tag(kDisallowTag) = Adopt(new bool(false));
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, true);
|
||||
@@ -161,7 +226,7 @@ TEST_F(GateCalculatorTest, DisallowByALLOWSidePacketSetToFalse) {
|
||||
input_stream: "test_input"
|
||||
output_stream: "test_output"
|
||||
)");
|
||||
runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false));
|
||||
runner()->MutableSidePackets()->Tag(kAllowTag) = Adopt(new bool(false));
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, true);
|
||||
@@ -179,7 +244,7 @@ TEST_F(GateCalculatorTest, DisallowByDISALLOWSidePacketSetToTrue) {
|
||||
input_stream: "test_input"
|
||||
output_stream: "test_output"
|
||||
)");
|
||||
runner()->MutableSidePackets()->Tag("DISALLOW") = Adopt(new bool(true));
|
||||
runner()->MutableSidePackets()->Tag(kDisallowTag) = Adopt(new bool(true));
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, true);
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/core/graph_profile_calculator.pb.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/api2/packet.h"
|
||||
#include "mediapipe/framework/api2/port.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_profile.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// This calculator periodically copies the GraphProfile from
|
||||
// mediapipe::GraphProfiler::CaptureProfile to the "PROFILE" output stream.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "GraphProfileCalculator"
|
||||
// output_stream: "FRAME:any_frame"
|
||||
// output_stream: "PROFILE:graph_profile"
|
||||
// }
|
||||
//
|
||||
class GraphProfileCalculator : public Node {
|
||||
public:
|
||||
static constexpr Input<AnyType>::Multiple kFrameIn{"FRAME"};
|
||||
static constexpr Output<GraphProfile> kProfileOut{"PROFILE"};
|
||||
|
||||
MEDIAPIPE_NODE_CONTRACT(kFrameIn, kProfileOut);
|
||||
|
||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
auto options = cc->Options<::mediapipe::GraphProfileCalculatorOptions>();
|
||||
|
||||
if (prev_profile_ts_ == Timestamp::Unset() ||
|
||||
cc->InputTimestamp() - prev_profile_ts_ >= options.profile_interval()) {
|
||||
prev_profile_ts_ = cc->InputTimestamp();
|
||||
GraphProfile result;
|
||||
MP_RETURN_IF_ERROR(cc->GetProfilingContext()->CaptureProfile(&result));
|
||||
kProfileOut(cc).Send(result);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
Timestamp prev_profile_ts_;
|
||||
};
|
||||
|
||||
MEDIAPIPE_REGISTER_NODE(GraphProfileCalculator);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option objc_class_prefix = "MediaPipe";
|
||||
|
||||
message GraphProfileCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional GraphProfileCalculatorOptions ext = 367481815;
|
||||
}
|
||||
|
||||
// The interval in microseconds between successive reported GraphProfiles.
|
||||
optional int64 profile_interval = 1 [default = 1000000];
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "absl/time/time.h"
|
||||
#include "mediapipe/framework/calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_profile.pb.h"
|
||||
#include "mediapipe/framework/deps/clock.h"
|
||||
#include "mediapipe/framework/deps/message_matchers.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/proto_ns.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/port/threadpool.h"
|
||||
#include "mediapipe/framework/tool/simulation_clock_executor.h"
|
||||
|
||||
// Tests for GraphProfileCalculator.
|
||||
using testing::ElementsAre;
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
constexpr char kClockTag[] = "CLOCK";
|
||||
|
||||
using mediapipe::Clock;
|
||||
|
||||
// A Calculator with a fixed Process call latency.
|
||||
class SleepCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->InputSidePackets().Tag(kClockTag).Set<std::shared_ptr<Clock>>();
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
|
||||
cc->SetTimestampOffset(TimestampDiff(0));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status Open(CalculatorContext* cc) final {
|
||||
clock_ =
|
||||
cc->InputSidePackets().Tag(kClockTag).Get<std::shared_ptr<Clock>>();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
clock_->Sleep(absl::Milliseconds(5));
|
||||
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
std::shared_ptr<::mediapipe::Clock> clock_ = nullptr;
|
||||
};
|
||||
REGISTER_CALCULATOR(SleepCalculator);
|
||||
|
||||
// Tests showing GraphProfileCalculator reporting GraphProfile output packets.
|
||||
class GraphProfileCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUpProfileGraph() {
|
||||
ASSERT_TRUE(proto_ns::TextFormat::ParseFromString(R"(
|
||||
input_stream: "input_packets_0"
|
||||
node {
|
||||
calculator: 'SleepCalculator'
|
||||
input_side_packet: 'CLOCK:sync_clock'
|
||||
input_stream: 'input_packets_0'
|
||||
output_stream: 'output_packets_1'
|
||||
}
|
||||
node {
|
||||
calculator: "GraphProfileCalculator"
|
||||
options: {
|
||||
[mediapipe.GraphProfileCalculatorOptions.ext]: {
|
||||
profile_interval: 25000
|
||||
}
|
||||
}
|
||||
input_stream: "FRAME:output_packets_1"
|
||||
output_stream: "PROFILE:output_packets_0"
|
||||
}
|
||||
)",
|
||||
&graph_config_));
|
||||
}
|
||||
|
||||
static Packet PacketAt(int64 ts) {
|
||||
return Adopt(new int64(999)).At(Timestamp(ts));
|
||||
}
|
||||
static Packet None() { return Packet().At(Timestamp::OneOverPostStream()); }
|
||||
static bool IsNone(const Packet& packet) {
|
||||
return packet.Timestamp() == Timestamp::OneOverPostStream();
|
||||
}
|
||||
// Return the values of the timestamps of a vector of Packets.
|
||||
static std::vector<int64> TimestampValues(
|
||||
const std::vector<Packet>& packets) {
|
||||
std::vector<int64> result;
|
||||
for (const Packet& p : packets) {
|
||||
result.push_back(p.Timestamp().Value());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Runs a CalculatorGraph with a series of packet sets.
|
||||
// Returns a vector of packets from each graph output stream.
|
||||
void RunGraph(const std::vector<std::vector<Packet>>& input_sets,
|
||||
std::vector<Packet>* output_packets) {
|
||||
// Register output packet observers.
|
||||
tool::AddVectorSink("output_packets_0", &graph_config_, output_packets);
|
||||
|
||||
// Start running the graph.
|
||||
std::shared_ptr<SimulationClockExecutor> executor(
|
||||
new SimulationClockExecutor(3 /*num_threads*/));
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.SetExecutor("", executor));
|
||||
graph.profiler()->SetClock(executor->GetClock());
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config_));
|
||||
executor->GetClock()->ThreadStart();
|
||||
MP_ASSERT_OK(graph.StartRun({
|
||||
{"sync_clock",
|
||||
Adopt(new std::shared_ptr<::mediapipe::Clock>(executor->GetClock()))},
|
||||
}));
|
||||
|
||||
// Send each packet to the graph in the specified order.
|
||||
for (int t = 0; t < input_sets.size(); t++) {
|
||||
const std::vector<Packet>& input_set = input_sets[t];
|
||||
for (int i = 0; i < input_set.size(); i++) {
|
||||
const Packet& packet = input_set[i];
|
||||
if (!IsNone(packet)) {
|
||||
MP_EXPECT_OK(graph.AddPacketToInputStream(
|
||||
absl::StrCat("input_packets_", i), packet));
|
||||
}
|
||||
executor->GetClock()->Sleep(absl::Milliseconds(10));
|
||||
}
|
||||
}
|
||||
MP_ASSERT_OK(graph.CloseAllInputStreams());
|
||||
executor->GetClock()->Sleep(absl::Milliseconds(100));
|
||||
executor->GetClock()->ThreadFinish();
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
CalculatorGraphConfig graph_config_;
|
||||
};
|
||||
|
||||
TEST_F(GraphProfileCalculatorTest, GraphProfile) {
|
||||
SetUpProfileGraph();
|
||||
auto profiler_config = graph_config_.mutable_profiler_config();
|
||||
profiler_config->set_enable_profiler(true);
|
||||
profiler_config->set_trace_enabled(false);
|
||||
profiler_config->set_trace_log_disabled(true);
|
||||
profiler_config->set_enable_stream_latency(true);
|
||||
profiler_config->set_calculator_filter(".*Calculator");
|
||||
|
||||
// Run the graph with a series of packet sets.
|
||||
std::vector<std::vector<Packet>> input_sets = {
|
||||
{PacketAt(10000)}, //
|
||||
{PacketAt(20000)}, //
|
||||
{PacketAt(30000)}, //
|
||||
{PacketAt(40000)},
|
||||
};
|
||||
std::vector<Packet> output_packets;
|
||||
RunGraph(input_sets, &output_packets);
|
||||
|
||||
// Validate the output packets.
|
||||
EXPECT_THAT(TimestampValues(output_packets), //
|
||||
ElementsAre(10000, 40000));
|
||||
|
||||
GraphProfile expected_profile =
|
||||
mediapipe::ParseTextProtoOrDie<GraphProfile>(R"pb(
|
||||
calculator_profiles {
|
||||
name: "GraphProfileCalculator"
|
||||
open_runtime: 0
|
||||
process_runtime { total: 0 count: 3 }
|
||||
process_input_latency { total: 15000 count: 3 }
|
||||
process_output_latency { total: 15000 count: 3 }
|
||||
input_stream_profiles {
|
||||
name: "output_packets_1"
|
||||
back_edge: false
|
||||
latency { total: 0 count: 3 }
|
||||
}
|
||||
}
|
||||
calculator_profiles {
|
||||
name: "SleepCalculator"
|
||||
open_runtime: 0
|
||||
process_runtime { total: 15000 count: 3 }
|
||||
process_input_latency { total: 0 count: 3 }
|
||||
process_output_latency { total: 15000 count: 3 }
|
||||
input_stream_profiles {
|
||||
name: "input_packets_0"
|
||||
back_edge: false
|
||||
latency { total: 0 count: 3 }
|
||||
}
|
||||
})pb");
|
||||
|
||||
EXPECT_THAT(output_packets[1].Get<GraphProfile>(),
|
||||
mediapipe::EqualsProto(expected_profile));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -29,6 +29,9 @@
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
constexpr char kMinuendTag[] = "MINUEND";
|
||||
constexpr char kSubtrahendTag[] = "SUBTRAHEND";
|
||||
|
||||
// A 3x4 Matrix of random integers in [0,1000).
|
||||
const char kMatrixText[] =
|
||||
"rows: 3\n"
|
||||
@@ -104,12 +107,13 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
|
||||
CalculatorRunner runner(node_config);
|
||||
Matrix* side_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText, side_matrix);
|
||||
runner.MutableSidePackets()->Tag("SUBTRAHEND") = Adopt(side_matrix);
|
||||
runner.MutableSidePackets()->Tag(kSubtrahendTag) = Adopt(side_matrix);
|
||||
|
||||
Matrix* input_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText2, input_matrix);
|
||||
runner.MutableInputs()->Tag("MINUEND").packets.push_back(
|
||||
Adopt(input_matrix).At(Timestamp(0)));
|
||||
runner.MutableInputs()
|
||||
->Tag(kMinuendTag)
|
||||
.packets.push_back(Adopt(input_matrix).At(Timestamp(0)));
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
EXPECT_EQ(1, runner.Outputs().Index(0).packets.size());
|
||||
@@ -133,12 +137,12 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
|
||||
CalculatorRunner runner(node_config);
|
||||
Matrix* side_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText, side_matrix);
|
||||
runner.MutableSidePackets()->Tag("MINUEND") = Adopt(side_matrix);
|
||||
runner.MutableSidePackets()->Tag(kMinuendTag) = Adopt(side_matrix);
|
||||
|
||||
Matrix* input_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText2, input_matrix);
|
||||
runner.MutableInputs()
|
||||
->Tag("SUBTRAHEND")
|
||||
->Tag(kSubtrahendTag)
|
||||
.packets.push_back(Adopt(input_matrix).At(Timestamp(0)));
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
@@ -17,6 +17,9 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kPresenceTag[] = "PRESENCE";
|
||||
constexpr char kPacketTag[] = "PACKET";
|
||||
|
||||
// For each non empty input packet, emits a single output packet containing a
|
||||
// boolean value "true", "false" in response to empty packets (a.k.a. timestamp
|
||||
// bound updates) This can be used to "flag" the presence of an arbitrary packet
|
||||
@@ -58,8 +61,8 @@ namespace mediapipe {
|
||||
class PacketPresenceCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag("PACKET").SetAny();
|
||||
cc->Outputs().Tag("PRESENCE").Set<bool>();
|
||||
cc->Inputs().Tag(kPacketTag).SetAny();
|
||||
cc->Outputs().Tag(kPresenceTag).Set<bool>();
|
||||
// Process() function is invoked in response to input stream timestamp
|
||||
// bound updates.
|
||||
cc->SetProcessTimestampBounds(true);
|
||||
@@ -73,8 +76,8 @@ class PacketPresenceCalculator : public CalculatorBase {
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
cc->Outputs()
|
||||
.Tag("PRESENCE")
|
||||
.AddPacket(MakePacket<bool>(!cc->Inputs().Tag("PACKET").IsEmpty())
|
||||
.Tag(kPresenceTag)
|
||||
.AddPacket(MakePacket<bool>(!cc->Inputs().Tag(kPacketTag).IsEmpty())
|
||||
.At(cc->InputTimestamp()));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -39,6 +39,11 @@ namespace mediapipe {
|
||||
|
||||
REGISTER_CALCULATOR(PacketResamplerCalculator);
|
||||
namespace {
|
||||
|
||||
constexpr char kSeedTag[] = "SEED";
|
||||
constexpr char kVideoHeaderTag[] = "VIDEO_HEADER";
|
||||
constexpr char kOptionsTag[] = "OPTIONS";
|
||||
|
||||
// Returns a TimestampDiff (assuming microseconds) corresponding to the
|
||||
// given time in seconds.
|
||||
TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
||||
@@ -50,16 +55,16 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
||||
absl::Status PacketResamplerCalculator::GetContract(CalculatorContract* cc) {
|
||||
const auto& resampler_options =
|
||||
cc->Options<PacketResamplerCalculatorOptions>();
|
||||
if (cc->InputSidePackets().HasTag("OPTIONS")) {
|
||||
cc->InputSidePackets().Tag("OPTIONS").Set<CalculatorOptions>();
|
||||
if (cc->InputSidePackets().HasTag(kOptionsTag)) {
|
||||
cc->InputSidePackets().Tag(kOptionsTag).Set<CalculatorOptions>();
|
||||
}
|
||||
CollectionItemId input_data_id = cc->Inputs().GetId("DATA", 0);
|
||||
if (!input_data_id.IsValid()) {
|
||||
input_data_id = cc->Inputs().GetId("", 0);
|
||||
}
|
||||
cc->Inputs().Get(input_data_id).SetAny();
|
||||
if (cc->Inputs().HasTag("VIDEO_HEADER")) {
|
||||
cc->Inputs().Tag("VIDEO_HEADER").Set<VideoHeader>();
|
||||
if (cc->Inputs().HasTag(kVideoHeaderTag)) {
|
||||
cc->Inputs().Tag(kVideoHeaderTag).Set<VideoHeader>();
|
||||
}
|
||||
|
||||
CollectionItemId output_data_id = cc->Outputs().GetId("DATA", 0);
|
||||
@@ -67,15 +72,15 @@ absl::Status PacketResamplerCalculator::GetContract(CalculatorContract* cc) {
|
||||
output_data_id = cc->Outputs().GetId("", 0);
|
||||
}
|
||||
cc->Outputs().Get(output_data_id).SetSameAs(&cc->Inputs().Get(input_data_id));
|
||||
if (cc->Outputs().HasTag("VIDEO_HEADER")) {
|
||||
cc->Outputs().Tag("VIDEO_HEADER").Set<VideoHeader>();
|
||||
if (cc->Outputs().HasTag(kVideoHeaderTag)) {
|
||||
cc->Outputs().Tag(kVideoHeaderTag).Set<VideoHeader>();
|
||||
}
|
||||
|
||||
if (resampler_options.jitter() != 0.0) {
|
||||
RET_CHECK_GT(resampler_options.jitter(), 0.0);
|
||||
RET_CHECK_LE(resampler_options.jitter(), 1.0);
|
||||
RET_CHECK(cc->InputSidePackets().HasTag("SEED"));
|
||||
cc->InputSidePackets().Tag("SEED").Set<std::string>();
|
||||
RET_CHECK(cc->InputSidePackets().HasTag(kSeedTag));
|
||||
cc->InputSidePackets().Tag(kSeedTag).Set<std::string>();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
@@ -143,9 +148,9 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
|
||||
|
||||
absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
|
||||
if (cc->InputTimestamp() == Timestamp::PreStream() &&
|
||||
cc->Inputs().UsesTags() && cc->Inputs().HasTag("VIDEO_HEADER") &&
|
||||
!cc->Inputs().Tag("VIDEO_HEADER").IsEmpty()) {
|
||||
video_header_ = cc->Inputs().Tag("VIDEO_HEADER").Get<VideoHeader>();
|
||||
cc->Inputs().UsesTags() && cc->Inputs().HasTag(kVideoHeaderTag) &&
|
||||
!cc->Inputs().Tag(kVideoHeaderTag).IsEmpty()) {
|
||||
video_header_ = cc->Inputs().Tag(kVideoHeaderTag).Get<VideoHeader>();
|
||||
video_header_.frame_rate = frame_rate_;
|
||||
if (cc->Inputs().Get(input_data_id_).IsEmpty()) {
|
||||
return absl::OkStatus();
|
||||
@@ -234,7 +239,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::InvalidArgumentError(
|
||||
@@ -357,7 +362,7 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Open(
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::InvalidArgumentError(
|
||||
@@ -504,7 +509,7 @@ absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::InvalidArgumentError(
|
||||
@@ -635,9 +640,9 @@ absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
|
||||
base_timestamp_ +
|
||||
TimestampDiffFromSeconds(first_index / calculator_->frame_rate_);
|
||||
}
|
||||
if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
|
||||
if (cc->Outputs().UsesTags() && cc->Outputs().HasTag(kVideoHeaderTag)) {
|
||||
cc->Outputs()
|
||||
.Tag("VIDEO_HEADER")
|
||||
.Tag(kVideoHeaderTag)
|
||||
.Add(new VideoHeader(calculator_->video_header_),
|
||||
Timestamp::PreStream());
|
||||
}
|
||||
|
||||
@@ -32,6 +32,12 @@ namespace mediapipe {
|
||||
|
||||
using ::testing::ElementsAre;
|
||||
namespace {
|
||||
|
||||
constexpr char kOptionsTag[] = "OPTIONS";
|
||||
constexpr char kSeedTag[] = "SEED";
|
||||
constexpr char kVideoHeaderTag[] = "VIDEO_HEADER";
|
||||
constexpr char kDataTag[] = "DATA";
|
||||
|
||||
// A simple version of CalculatorRunner with built-in convenience
|
||||
// methods for setting inputs from a vector and checking outputs
|
||||
// against expected outputs (both timestamps and contents).
|
||||
@@ -464,7 +470,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
||||
)pb"));
|
||||
|
||||
for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(
|
||||
runner.MutableInputs()->Tag(kDataTag).packets.push_back(
|
||||
Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts)));
|
||||
}
|
||||
VideoHeader video_header_in;
|
||||
@@ -474,16 +480,16 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
||||
video_header_in.duration = 1.0;
|
||||
video_header_in.format = ImageFormat::SRGB;
|
||||
runner.MutableInputs()
|
||||
->Tag("VIDEO_HEADER")
|
||||
->Tag(kVideoHeaderTag)
|
||||
.packets.push_back(
|
||||
Adopt(new VideoHeader(video_header_in)).At(Timestamp::PreStream()));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
ASSERT_EQ(1, runner.Outputs().Tag("VIDEO_HEADER").packets.size());
|
||||
ASSERT_EQ(1, runner.Outputs().Tag(kVideoHeaderTag).packets.size());
|
||||
EXPECT_EQ(Timestamp::PreStream(),
|
||||
runner.Outputs().Tag("VIDEO_HEADER").packets[0].Timestamp());
|
||||
runner.Outputs().Tag(kVideoHeaderTag).packets[0].Timestamp());
|
||||
const VideoHeader& video_header_out =
|
||||
runner.Outputs().Tag("VIDEO_HEADER").packets[0].Get<VideoHeader>();
|
||||
runner.Outputs().Tag(kVideoHeaderTag).packets[0].Get<VideoHeader>();
|
||||
EXPECT_EQ(video_header_in.width, video_header_out.width);
|
||||
EXPECT_EQ(video_header_in.height, video_header_out.height);
|
||||
EXPECT_DOUBLE_EQ(50.0, video_header_out.frame_rate);
|
||||
@@ -725,7 +731,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
[mediapipe.PacketResamplerCalculatorOptions.ext] {
|
||||
frame_rate: 30
|
||||
})pb"));
|
||||
runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
|
||||
runner.MutableSidePackets()->Tag(kOptionsTag) = Adopt(options);
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
EXPECT_EQ(6, runner.Outputs().Index(0).packets.size());
|
||||
@@ -740,7 +746,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
frame_rate: 30
|
||||
base_timestamp: 0
|
||||
})pb"));
|
||||
runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
|
||||
runner.MutableSidePackets()->Tag(kOptionsTag) = Adopt(options);
|
||||
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
@@ -29,6 +29,8 @@
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
constexpr char kPeriodTag[] = "PERIOD";
|
||||
|
||||
// A simple version of CalculatorRunner with built-in convenience methods for
|
||||
// setting inputs from a vector and checking outputs against a vector of
|
||||
// expected outputs.
|
||||
@@ -121,7 +123,7 @@ TEST(PacketThinnerCalculatorTest, ASyncUniformStreamThinningTestBySidePacket) {
|
||||
|
||||
SimpleRunner runner(node);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
|
||||
runner.MutableSidePackets()->Tag(kPeriodTag) = MakePacket<int64>(5);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 8, 14};
|
||||
@@ -160,7 +162,7 @@ TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTestBySidePacket1) {
|
||||
|
||||
SimpleRunner runner(node);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
|
||||
runner.MutableSidePackets()->Tag(kPeriodTag) = MakePacket<int64>(5);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
|
||||
|
||||
@@ -39,6 +39,8 @@ using ::testing::Pair;
|
||||
using ::testing::Value;
|
||||
namespace {
|
||||
|
||||
constexpr char kDisallowTag[] = "DISALLOW";
|
||||
|
||||
// Returns the timestamp values for a vector of Packets.
|
||||
// TODO: puth this kind of test util in a common place.
|
||||
std::vector<int64> TimestampValues(const std::vector<Packet>& packets) {
|
||||
@@ -702,14 +704,14 @@ class DroppingGateCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
cc->Inputs().Tag("DISALLOW").Set<bool>();
|
||||
cc->Inputs().Tag(kDisallowTag).Set<bool>();
|
||||
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
if (!cc->Inputs().Index(0).IsEmpty() &&
|
||||
!cc->Inputs().Tag("DISALLOW").Get<bool>()) {
|
||||
!cc->Inputs().Tag(kDisallowTag).Get<bool>()) {
|
||||
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
|
||||
}
|
||||
return absl::OkStatus();
|
||||
|
||||
@@ -41,11 +41,14 @@
|
||||
// }
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kEncodedTag[] = "ENCODED";
|
||||
constexpr char kFloatVectorTag[] = "FLOAT_VECTOR";
|
||||
|
||||
class QuantizeFloatVectorCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
|
||||
cc->Outputs().Tag("ENCODED").Set<std::string>();
|
||||
cc->Inputs().Tag(kFloatVectorTag).Set<std::vector<float>>();
|
||||
cc->Outputs().Tag(kEncodedTag).Set<std::string>();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -70,7 +73,7 @@ class QuantizeFloatVectorCalculator : public CalculatorBase {
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
const std::vector<float>& float_vector =
|
||||
cc->Inputs().Tag("FLOAT_VECTOR").Value().Get<std::vector<float>>();
|
||||
cc->Inputs().Tag(kFloatVectorTag).Value().Get<std::vector<float>>();
|
||||
int feature_size = float_vector.size();
|
||||
std::string encoded_features;
|
||||
encoded_features.reserve(feature_size);
|
||||
@@ -86,8 +89,10 @@ class QuantizeFloatVectorCalculator : public CalculatorBase {
|
||||
(old_value - min_quantized_value_) * (255.0 / range_));
|
||||
encoded_features += encoded;
|
||||
}
|
||||
cc->Outputs().Tag("ENCODED").AddPacket(
|
||||
MakePacket<std::string>(encoded_features).At(cc->InputTimestamp()));
|
||||
cc->Outputs()
|
||||
.Tag(kEncodedTag)
|
||||
.AddPacket(
|
||||
MakePacket<std::string>(encoded_features).At(cc->InputTimestamp()));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,9 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kEncodedTag[] = "ENCODED";
|
||||
constexpr char kFloatVectorTag[] = "FLOAT_VECTOR";
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
@@ -40,7 +43,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
->Tag(kFloatVectorTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
@@ -67,7 +70,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
->Tag(kFloatVectorTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
@@ -94,7 +97,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
->Tag(kFloatVectorTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
@@ -121,11 +124,12 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
->Tag(kFloatVectorTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
|
||||
const std::vector<Packet>& outputs =
|
||||
runner.Outputs().Tag(kEncodedTag).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_TRUE(outputs[0].Get<std::string>().empty());
|
||||
EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
|
||||
@@ -147,11 +151,12 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f};
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
->Tag(kFloatVectorTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
|
||||
const std::vector<Packet>& outputs =
|
||||
runner.Outputs().Tag(kEncodedTag).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
const std::string& result = outputs[0].Get<std::string>();
|
||||
ASSERT_FALSE(result.empty());
|
||||
@@ -185,11 +190,12 @@ TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> vector = {-65.0f, 65.0f};
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
->Tag(kFloatVectorTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
|
||||
const std::vector<Packet>& outputs =
|
||||
runner.Outputs().Tag(kEncodedTag).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
const std::string& result = outputs[0].Get<std::string>();
|
||||
ASSERT_FALSE(result.empty());
|
||||
|
||||
@@ -23,6 +23,9 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kAllowTag[] = "ALLOW";
|
||||
constexpr char kMaxInFlightTag[] = "MAX_IN_FLIGHT";
|
||||
|
||||
// RealTimeFlowLimiterCalculator is used to limit the number of pipelined
|
||||
// processing operations in a section of the graph.
|
||||
//
|
||||
@@ -86,11 +89,11 @@ class RealTimeFlowLimiterCalculator : public CalculatorBase {
|
||||
cc->Outputs().Get("", i).SetSameAs(&(cc->Inputs().Get("", i)));
|
||||
}
|
||||
cc->Inputs().Get("FINISHED", 0).SetAny();
|
||||
if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
|
||||
cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Set<int>();
|
||||
if (cc->InputSidePackets().HasTag(kMaxInFlightTag)) {
|
||||
cc->InputSidePackets().Tag(kMaxInFlightTag).Set<int>();
|
||||
}
|
||||
if (cc->Outputs().HasTag("ALLOW")) {
|
||||
cc->Outputs().Tag("ALLOW").Set<bool>();
|
||||
if (cc->Outputs().HasTag(kAllowTag)) {
|
||||
cc->Outputs().Tag(kAllowTag).Set<bool>();
|
||||
}
|
||||
|
||||
cc->SetInputStreamHandler("ImmediateInputStreamHandler");
|
||||
@@ -101,8 +104,8 @@ class RealTimeFlowLimiterCalculator : public CalculatorBase {
|
||||
absl::Status Open(CalculatorContext* cc) final {
|
||||
finished_id_ = cc->Inputs().GetId("FINISHED", 0);
|
||||
max_in_flight_ = 1;
|
||||
if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
|
||||
max_in_flight_ = cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Get<int>();
|
||||
if (cc->InputSidePackets().HasTag(kMaxInFlightTag)) {
|
||||
max_in_flight_ = cc->InputSidePackets().Tag(kMaxInFlightTag).Get<int>();
|
||||
}
|
||||
RET_CHECK_GE(max_in_flight_, 1);
|
||||
num_in_flight_ = 0;
|
||||
|
||||
@@ -33,6 +33,9 @@
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kFinishedTag[] = "FINISHED";
|
||||
|
||||
// A simple Semaphore for synchronizing test threads.
|
||||
class AtomicSemaphore {
|
||||
public:
|
||||
@@ -112,7 +115,7 @@ TEST(RealTimeFlowLimiterCalculator, BasicTest) {
|
||||
Timestamp timestamp =
|
||||
Timestamp((i + 1) * Timestamp::kTimestampUnitsPerSecond);
|
||||
runner.MutableInputs()
|
||||
->Tag("FINISHED")
|
||||
->Tag(kFinishedTag)
|
||||
.packets.push_back(MakePacket<bool>(true).At(timestamp));
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,8 @@ namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kPacketOffsetTag[] = "PACKET_OFFSET";
|
||||
|
||||
// Adds packets containing integers equal to their original timestamp.
|
||||
void AddPackets(CalculatorRunner* runner) {
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
@@ -111,7 +113,7 @@ TEST(SequenceShiftCalculatorTest, SidePacketOffset) {
|
||||
|
||||
CalculatorRunner runner(node);
|
||||
AddPackets(&runner);
|
||||
runner.MutableSidePackets()->Tag("PACKET_OFFSET") = Adopt(new int(-2));
|
||||
runner.MutableSidePackets()->Tag(kPacketOffsetTag) = Adopt(new int(-2));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& input_packets =
|
||||
runner.MutableInputs()->Index(0).packets;
|
||||
|
||||
@@ -661,3 +661,138 @@ cc_test(
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "affine_transformation",
|
||||
hdrs = ["affine_transformation.h"],
|
||||
deps = ["@com_google_absl//absl/status:statusor"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "affine_transformation_runner_gl",
|
||||
srcs = ["affine_transformation_runner_gl.cc"],
|
||||
hdrs = ["affine_transformation_runner_gl.h"],
|
||||
deps = [
|
||||
":affine_transformation",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||
"//mediapipe/gpu:shader_util",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "affine_transformation_runner_opencv",
|
||||
srcs = ["affine_transformation_runner_opencv.cc"],
|
||||
hdrs = ["affine_transformation_runner_opencv.h"],
|
||||
deps = [
|
||||
":affine_transformation",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "warp_affine_calculator_proto",
|
||||
srcs = ["warp_affine_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/gpu:gpu_origin_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "warp_affine_calculator",
|
||||
srcs = ["warp_affine_calculator.cc"],
|
||||
hdrs = ["warp_affine_calculator.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":affine_transformation",
|
||||
":affine_transformation_runner_opencv",
|
||||
":warp_affine_calculator_cc_proto",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/api2:port",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
":affine_transformation_runner_gl",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "warp_affine_calculator_test",
|
||||
srcs = ["warp_affine_calculator_test.cc"],
|
||||
data = [
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/input.jpg",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_border_zero.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_keep_aspect.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_keep_aspect_border_zero.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation_border_zero.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_keep_aspect.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_keep_aspect_border_zero.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation_border_zero.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
|
||||
],
|
||||
tags = ["desktop_only_test"],
|
||||
deps = [
|
||||
":affine_transformation",
|
||||
":warp_affine_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_converter",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_utils",
|
||||
"//mediapipe/calculators/util:from_image_calculator",
|
||||
"//mediapipe/calculators/util:to_image_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
|
||||
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_H_
|
||||
#define MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_H_
|
||||
|
||||
#include <array>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class AffineTransformation {
|
||||
public:
|
||||
// Pixel extrapolation method.
|
||||
// When converting image to tensor it may happen that tensor needs to read
|
||||
// pixels outside image boundaries. Border mode helps to specify how such
|
||||
// pixels will be calculated.
|
||||
enum class BorderMode { kZero, kReplicate };
|
||||
|
||||
struct Size {
|
||||
int width;
|
||||
int height;
|
||||
};
|
||||
|
||||
template <typename InputT, typename OutputT>
|
||||
class Runner {
|
||||
public:
|
||||
virtual ~Runner() = default;
|
||||
|
||||
// Transforms input into output using @matrix as following:
|
||||
// output(x, y) = input(matrix[0] * x + matrix[1] * y + matrix[3],
|
||||
// matrix[4] * x + matrix[5] * y + matrix[7])
|
||||
// where x and y ranges are defined by @output_size.
|
||||
virtual absl::StatusOr<OutputT> Run(const InputT& input,
|
||||
const std::array<float, 16>& matrix,
|
||||
const Size& output_size,
|
||||
BorderMode border_mode) = 0;
|
||||
};
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_H_
|
||||
@@ -0,0 +1,354 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/image/affine_transformation_runner_gl.h"
|
||||
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "Eigen/Geometry"
|
||||
#include "Eigen/LU"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/image/affine_transformation.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#include "mediapipe/gpu/gl_simple_shaders.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "mediapipe/gpu/gpu_origin.pb.h"
|
||||
#include "mediapipe/gpu/shader_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
using mediapipe::GlCalculatorHelper;
|
||||
using mediapipe::GlhCreateProgram;
|
||||
using mediapipe::GlTexture;
|
||||
using mediapipe::GpuBuffer;
|
||||
using mediapipe::GpuOrigin;
|
||||
|
||||
bool IsMatrixVerticalFlipNeeded(GpuOrigin::Mode gpu_origin) {
|
||||
switch (gpu_origin) {
|
||||
case GpuOrigin::DEFAULT:
|
||||
case GpuOrigin::CONVENTIONAL:
|
||||
#ifdef __APPLE__
|
||||
return false;
|
||||
#else
|
||||
return true;
|
||||
#endif // __APPLE__
|
||||
case GpuOrigin::TOP_LEFT:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
#define GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED 0
|
||||
#else
|
||||
#define GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED 1
|
||||
#endif // __APPLE__
|
||||
|
||||
bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context) {
|
||||
return gl_context.gl_major_version() > 3 ||
|
||||
(gl_context.gl_major_version() == 3 &&
|
||||
gl_context.gl_minor_version() >= 2);
|
||||
}
|
||||
|
||||
constexpr int kAttribVertex = 0;
|
||||
constexpr int kAttribTexturePosition = 1;
|
||||
constexpr int kNumAttributes = 2;
|
||||
|
||||
class GlTextureWarpAffineRunner
|
||||
: public AffineTransformation::Runner<GpuBuffer,
|
||||
std::unique_ptr<GpuBuffer>> {
|
||||
public:
|
||||
GlTextureWarpAffineRunner(std::shared_ptr<GlCalculatorHelper> gl_helper,
|
||||
GpuOrigin::Mode gpu_origin)
|
||||
: gl_helper_(gl_helper), gpu_origin_(gpu_origin) {}
|
||||
absl::Status Init() {
|
||||
return gl_helper_->RunInGlContext([this]() -> absl::Status {
|
||||
const GLint attr_location[kNumAttributes] = {
|
||||
kAttribVertex,
|
||||
kAttribTexturePosition,
|
||||
};
|
||||
const GLchar* attr_name[kNumAttributes] = {
|
||||
"position",
|
||||
"texture_coordinate",
|
||||
};
|
||||
|
||||
constexpr GLchar kVertShader[] = R"(
|
||||
in vec4 position;
|
||||
in mediump vec4 texture_coordinate;
|
||||
out mediump vec2 sample_coordinate;
|
||||
uniform mat4 transform_matrix;
|
||||
|
||||
void main() {
|
||||
gl_Position = position;
|
||||
vec4 tc = transform_matrix * texture_coordinate;
|
||||
sample_coordinate = tc.xy;
|
||||
}
|
||||
)";
|
||||
|
||||
constexpr GLchar kFragShader[] = R"(
|
||||
DEFAULT_PRECISION(mediump, float)
|
||||
in vec2 sample_coordinate;
|
||||
uniform sampler2D input_texture;
|
||||
|
||||
#ifdef GL_ES
|
||||
#define fragColor gl_FragColor
|
||||
#else
|
||||
out vec4 fragColor;
|
||||
#endif // defined(GL_ES);
|
||||
|
||||
void main() {
|
||||
vec4 color = texture2D(input_texture, sample_coordinate);
|
||||
#ifdef CUSTOM_ZERO_BORDER_MODE
|
||||
float out_of_bounds =
|
||||
float(sample_coordinate.x < 0.0 || sample_coordinate.x > 1.0 ||
|
||||
sample_coordinate.y < 0.0 || sample_coordinate.y > 1.0);
|
||||
color = mix(color, vec4(0.0, 0.0, 0.0, 0.0), out_of_bounds);
|
||||
#endif // defined(CUSTOM_ZERO_BORDER_MODE)
|
||||
fragColor = color;
|
||||
}
|
||||
)";
|
||||
|
||||
// Create program and set parameters.
|
||||
auto create_fn = [&](const std::string& vs,
|
||||
const std::string& fs) -> absl::StatusOr<Program> {
|
||||
GLuint program = 0;
|
||||
GlhCreateProgram(vs.c_str(), fs.c_str(), kNumAttributes, &attr_name[0],
|
||||
attr_location, &program);
|
||||
|
||||
RET_CHECK(program) << "Problem initializing warp affine program.";
|
||||
glUseProgram(program);
|
||||
glUniform1i(glGetUniformLocation(program, "input_texture"), 1);
|
||||
GLint matrix_id = glGetUniformLocation(program, "transform_matrix");
|
||||
return Program{.id = program, .matrix_id = matrix_id};
|
||||
};
|
||||
|
||||
const std::string vert_src =
|
||||
absl::StrCat(mediapipe::kMediaPipeVertexShaderPreamble, kVertShader);
|
||||
|
||||
const std::string frag_src = absl::StrCat(
|
||||
mediapipe::kMediaPipeFragmentShaderPreamble, kFragShader);
|
||||
|
||||
ASSIGN_OR_RETURN(program_, create_fn(vert_src, frag_src));
|
||||
|
||||
auto create_custom_zero_fn = [&]() -> absl::StatusOr<Program> {
|
||||
std::string custom_zero_border_mode_def = R"(
|
||||
#define CUSTOM_ZERO_BORDER_MODE
|
||||
)";
|
||||
const std::string frag_custom_zero_src =
|
||||
absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble,
|
||||
custom_zero_border_mode_def, kFragShader);
|
||||
return create_fn(vert_src, frag_custom_zero_src);
|
||||
};
|
||||
#if GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
||||
if (!IsGlClampToBorderSupported(gl_helper_->GetGlContext())) {
|
||||
ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
||||
}
|
||||
#else
|
||||
ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
||||
#endif // GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
||||
|
||||
glGenFramebuffers(1, &framebuffer_);
|
||||
|
||||
// vertex storage
|
||||
glGenBuffers(2, vbo_);
|
||||
glGenVertexArrays(1, &vao_);
|
||||
|
||||
// vbo 0
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]);
|
||||
glBufferData(GL_ARRAY_BUFFER, sizeof(mediapipe::kBasicSquareVertices),
|
||||
mediapipe::kBasicSquareVertices, GL_STATIC_DRAW);
|
||||
|
||||
// vbo 1
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]);
|
||||
glBufferData(GL_ARRAY_BUFFER, sizeof(mediapipe::kBasicTextureVertices),
|
||||
mediapipe::kBasicTextureVertices, GL_STATIC_DRAW);
|
||||
|
||||
glBindBuffer(GL_ARRAY_BUFFER, 0);
|
||||
|
||||
return absl::OkStatus();
|
||||
});
|
||||
}
|
||||
|
||||
absl::StatusOr<std::unique_ptr<GpuBuffer>> Run(
|
||||
const GpuBuffer& input, const std::array<float, 16>& matrix,
|
||||
const AffineTransformation::Size& size,
|
||||
AffineTransformation::BorderMode border_mode) override {
|
||||
std::unique_ptr<GpuBuffer> gpu_buffer;
|
||||
MP_RETURN_IF_ERROR(
|
||||
gl_helper_->RunInGlContext([this, &input, &matrix, &size, &border_mode,
|
||||
&gpu_buffer]() -> absl::Status {
|
||||
auto input_texture = gl_helper_->CreateSourceTexture(input);
|
||||
auto output_texture = gl_helper_->CreateDestinationTexture(
|
||||
size.width, size.height, input.format());
|
||||
|
||||
MP_RETURN_IF_ERROR(
|
||||
RunInternal(input_texture, matrix, border_mode, &output_texture));
|
||||
gpu_buffer = output_texture.GetFrame<GpuBuffer>();
|
||||
return absl::OkStatus();
|
||||
}));
|
||||
|
||||
return gpu_buffer;
|
||||
}
|
||||
|
||||
absl::Status RunInternal(const GlTexture& texture,
|
||||
const std::array<float, 16>& matrix,
|
||||
AffineTransformation::BorderMode border_mode,
|
||||
GlTexture* output) {
|
||||
glDisable(GL_DEPTH_TEST);
|
||||
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
|
||||
glViewport(0, 0, output->width(), output->height());
|
||||
|
||||
glActiveTexture(GL_TEXTURE0);
|
||||
glBindTexture(GL_TEXTURE_2D, output->name());
|
||||
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
|
||||
output->name(), 0);
|
||||
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(texture.target(), texture.name());
|
||||
|
||||
// a) Filtering.
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
|
||||
|
||||
// b) Clamping.
|
||||
std::optional<Program> program = program_;
|
||||
switch (border_mode) {
|
||||
case AffineTransformation::BorderMode::kReplicate: {
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
|
||||
break;
|
||||
}
|
||||
case AffineTransformation::BorderMode::kZero: {
|
||||
#if GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
||||
if (program_custom_zero_) {
|
||||
program = program_custom_zero_;
|
||||
} else {
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
|
||||
glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR,
|
||||
std::array<float, 4>{0.0f, 0.0f, 0.0f, 0.0f}.data());
|
||||
}
|
||||
#else
|
||||
RET_CHECK(program_custom_zero_)
|
||||
<< "Program must have been initialized.";
|
||||
program = program_custom_zero_;
|
||||
#endif // GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
||||
break;
|
||||
}
|
||||
}
|
||||
glUseProgram(program->id);
|
||||
|
||||
Eigen::Matrix<float, 4, 4, Eigen::RowMajor> eigen_mat(matrix.data());
|
||||
if (IsMatrixVerticalFlipNeeded(gpu_origin_)) {
|
||||
// @matrix describes affine transformation in terms of TOP LEFT origin, so
|
||||
// in some cases/on some platforms an extra flipping should be done before
|
||||
// and after.
|
||||
const Eigen::Matrix<float, 4, 4, Eigen::RowMajor> flip_y(
|
||||
{{1.0f, 0.0f, 0.0f, 0.0f},
|
||||
{0.0f, -1.0f, 0.0f, 1.0f},
|
||||
{0.0f, 0.0f, 1.0f, 0.0f},
|
||||
{0.0f, 0.0f, 0.0f, 1.0f}});
|
||||
eigen_mat = flip_y * eigen_mat * flip_y;
|
||||
}
|
||||
|
||||
// If GL context is ES2, then GL_FALSE must be used for 'transpose'
|
||||
// GLboolean in glUniformMatrix4fv, or else INVALID_VALUE error is reported.
|
||||
// Hence, transposing the matrix and always passing transposed.
|
||||
eigen_mat.transposeInPlace();
|
||||
glUniformMatrix4fv(program->matrix_id, 1, GL_FALSE, eigen_mat.data());
|
||||
|
||||
// vao
|
||||
glBindVertexArray(vao_);
|
||||
|
||||
// vbo 0
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]);
|
||||
glEnableVertexAttribArray(kAttribVertex);
|
||||
glVertexAttribPointer(kAttribVertex, 2, GL_FLOAT, 0, 0, nullptr);
|
||||
|
||||
// vbo 1
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]);
|
||||
glEnableVertexAttribArray(kAttribTexturePosition);
|
||||
glVertexAttribPointer(kAttribTexturePosition, 2, GL_FLOAT, 0, 0, nullptr);
|
||||
|
||||
// draw
|
||||
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
|
||||
|
||||
// Resetting to MediaPipe texture param defaults.
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
|
||||
|
||||
glDisableVertexAttribArray(kAttribVertex);
|
||||
glDisableVertexAttribArray(kAttribTexturePosition);
|
||||
glBindBuffer(GL_ARRAY_BUFFER, 0);
|
||||
glBindVertexArray(0);
|
||||
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(GL_TEXTURE_2D, 0);
|
||||
glActiveTexture(GL_TEXTURE0);
|
||||
glBindTexture(GL_TEXTURE_2D, 0);
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
~GlTextureWarpAffineRunner() override {
|
||||
gl_helper_->RunInGlContext([this]() {
|
||||
// Release OpenGL resources.
|
||||
if (framebuffer_ != 0) glDeleteFramebuffers(1, &framebuffer_);
|
||||
if (program_.id != 0) glDeleteProgram(program_.id);
|
||||
if (program_custom_zero_ && program_custom_zero_->id != 0) {
|
||||
glDeleteProgram(program_custom_zero_->id);
|
||||
}
|
||||
if (vao_ != 0) glDeleteVertexArrays(1, &vao_);
|
||||
glDeleteBuffers(2, vbo_);
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
struct Program {
|
||||
GLuint id;
|
||||
GLint matrix_id;
|
||||
};
|
||||
std::shared_ptr<GlCalculatorHelper> gl_helper_;
|
||||
GpuOrigin::Mode gpu_origin_;
|
||||
GLuint vao_ = 0;
|
||||
GLuint vbo_[2] = {0, 0};
|
||||
Program program_;
|
||||
std::optional<Program> program_custom_zero_;
|
||||
GLuint framebuffer_ = 0;
|
||||
};
|
||||
|
||||
#undef GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
||||
|
||||
} // namespace
|
||||
|
||||
absl::StatusOr<std::unique_ptr<
|
||||
AffineTransformation::Runner<GpuBuffer, std::unique_ptr<GpuBuffer>>>>
|
||||
CreateAffineTransformationGlRunner(
|
||||
std::shared_ptr<GlCalculatorHelper> gl_helper, GpuOrigin::Mode gpu_origin) {
|
||||
auto runner =
|
||||
absl::make_unique<GlTextureWarpAffineRunner>(gl_helper, gpu_origin);
|
||||
MP_RETURN_IF_ERROR(runner->Init());
|
||||
return runner;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_GL_H_
|
||||
#define MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_GL_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/image/affine_transformation.h"
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "mediapipe/gpu/gpu_origin.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
absl::StatusOr<std::unique_ptr<AffineTransformation::Runner<
|
||||
mediapipe::GpuBuffer, std::unique_ptr<mediapipe::GpuBuffer>>>>
|
||||
CreateAffineTransformationGlRunner(
|
||||
std::shared_ptr<mediapipe::GlCalculatorHelper> gl_helper,
|
||||
mediapipe::GpuOrigin::Mode gpu_origin);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_GL_H_
|
||||
@@ -0,0 +1,160 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/image/affine_transformation_runner_opencv.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/image/affine_transformation.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
cv::BorderTypes GetBorderModeForOpenCv(
|
||||
AffineTransformation::BorderMode border_mode) {
|
||||
switch (border_mode) {
|
||||
case AffineTransformation::BorderMode::kZero:
|
||||
return cv::BORDER_CONSTANT;
|
||||
case AffineTransformation::BorderMode::kReplicate:
|
||||
return cv::BORDER_REPLICATE;
|
||||
}
|
||||
}
|
||||
|
||||
class OpenCvRunner
|
||||
: public AffineTransformation::Runner<ImageFrame, ImageFrame> {
|
||||
public:
|
||||
absl::StatusOr<ImageFrame> Run(
|
||||
const ImageFrame& input, const std::array<float, 16>& matrix,
|
||||
const AffineTransformation::Size& size,
|
||||
AffineTransformation::BorderMode border_mode) override {
|
||||
// OpenCV warpAffine works in absolute coordinates, so the transfom (which
|
||||
// accepts and produces relative coordinates) should be adjusted to first
|
||||
// normalize coordinates and then scale them.
|
||||
// clang-format off
|
||||
cv::Matx44f normalize_dst_coordinate({
|
||||
1.0f / size.width, 0.0f, 0.0f, 0.0f,
|
||||
0.0f, 1.0f / size.height, 0.0f, 0.0f,
|
||||
0.0f, 0.0f, 1.0f, 0.0f,
|
||||
0.0f, 0.0f, 0.0f, 1.0f});
|
||||
cv::Matx44f scale_src_coordinate({
|
||||
1.0f * input.Width(), 0.0f, 0.0f, 0.0f,
|
||||
0.0f, 1.0f * input.Height(), 0.0f, 0.0f,
|
||||
0.0f, 0.0f, 1.0f, 0.0f,
|
||||
0.0f, 0.0f, 0.0f, 1.0f});
|
||||
// clang-format on
|
||||
cv::Matx44f adjust_dst_coordinate;
|
||||
cv::Matx44f adjust_src_coordinate;
|
||||
// TODO: update to always use accurate implementation.
|
||||
constexpr bool kOpenCvCompatibility = true;
|
||||
if (kOpenCvCompatibility) {
|
||||
adjust_dst_coordinate = normalize_dst_coordinate;
|
||||
adjust_src_coordinate = scale_src_coordinate;
|
||||
} else {
|
||||
// To do an accurate affine image transformation and make "on-cpu" and
|
||||
// "on-gpu" calculations aligned - extra offset is required to select
|
||||
// correct pixels.
|
||||
//
|
||||
// Each destination pixel corresponds to some pixels region from source
|
||||
// image.(In case of downscaling there can be more than one pixel.) The
|
||||
// offset for x and y is calculated in the way, so pixel in the middle of
|
||||
// the region is selected.
|
||||
//
|
||||
// For simplicity sake, let's consider downscaling from 100x50 to 10x10
|
||||
// without a rotation:
|
||||
// 1. Each destination pixel corresponds to 10x5 region
|
||||
// X range: [0, .. , 9]
|
||||
// Y range: [0, .. , 4]
|
||||
// 2. Considering we have __discrete__ pixels, the center of the region is
|
||||
// between (4, 2) and (5, 2) pixels, let's assume it's a "pixel"
|
||||
// (4.5, 2).
|
||||
// 3. When using the above as an offset for every pixel select while
|
||||
// downscaling, resulting pixels are:
|
||||
// (4.5, 2), (14.5, 2), .. , (94.5, 2)
|
||||
// (4.5, 7), (14.5, 7), .. , (94.5, 7)
|
||||
// ..
|
||||
// (4.5, 47), (14.5, 47), .., (94.5, 47)
|
||||
// instead of:
|
||||
// (0, 0), (10, 0), .. , (90, 0)
|
||||
// (0, 5), (10, 7), .. , (90, 5)
|
||||
// ..
|
||||
// (0, 45), (10, 45), .., (90, 45)
|
||||
// The latter looks shifted.
|
||||
//
|
||||
// Offsets are needed, so that __discrete__ pixel at (0, 0) corresponds to
|
||||
// the same pixel as would __non discrete__ pixel at (0.5, 0.5). Hence,
|
||||
// transformation matrix should shift coordinates by (0.5, 0.5) as the
|
||||
// very first step.
|
||||
//
|
||||
// Due to the above shift, transformed coordinates would be valid for
|
||||
// float coordinates where pixel (0, 0) spans [0.0, 1.0) x [0.0, 1.0).
|
||||
// T0 make it valid for __discrete__ pixels, transformation matrix should
|
||||
// shift coordinate by (-0.5f, -0.5f) as the very last step. (E.g. if we
|
||||
// get (0.5f, 0.5f), then it's (0, 0) __discrete__ pixel.)
|
||||
// clang-format off
|
||||
cv::Matx44f shift_dst({1.0f, 0.0f, 0.0f, 0.5f,
|
||||
0.0f, 1.0f, 0.0f, 0.5f,
|
||||
0.0f, 0.0f, 1.0f, 0.0f,
|
||||
0.0f, 0.0f, 0.0f, 1.0f});
|
||||
cv::Matx44f shift_src({1.0f, 0.0f, 0.0f, -0.5f,
|
||||
0.0f, 1.0f, 0.0f, -0.5f,
|
||||
0.0f, 0.0f, 1.0f, 0.0f,
|
||||
0.0f, 0.0f, 0.0f, 1.0f});
|
||||
// clang-format on
|
||||
adjust_dst_coordinate = normalize_dst_coordinate * shift_dst;
|
||||
adjust_src_coordinate = shift_src * scale_src_coordinate;
|
||||
}
|
||||
|
||||
cv::Matx44f transform(matrix.data());
|
||||
cv::Matx44f transform_absolute =
|
||||
adjust_src_coordinate * transform * adjust_dst_coordinate;
|
||||
|
||||
cv::Mat in_mat = formats::MatView(&input);
|
||||
|
||||
cv::Mat cv_affine_transform(2, 3, CV_32F);
|
||||
cv_affine_transform.at<float>(0, 0) = transform_absolute.val[0];
|
||||
cv_affine_transform.at<float>(0, 1) = transform_absolute.val[1];
|
||||
cv_affine_transform.at<float>(0, 2) = transform_absolute.val[3];
|
||||
cv_affine_transform.at<float>(1, 0) = transform_absolute.val[4];
|
||||
cv_affine_transform.at<float>(1, 1) = transform_absolute.val[5];
|
||||
cv_affine_transform.at<float>(1, 2) = transform_absolute.val[7];
|
||||
|
||||
ImageFrame out_image(input.Format(), size.width, size.height);
|
||||
cv::Mat out_mat = formats::MatView(&out_image);
|
||||
|
||||
cv::warpAffine(in_mat, out_mat, cv_affine_transform,
|
||||
cv::Size(out_mat.cols, out_mat.rows),
|
||||
/*flags=*/cv::INTER_LINEAR | cv::WARP_INVERSE_MAP,
|
||||
GetBorderModeForOpenCv(border_mode));
|
||||
|
||||
return out_image;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
absl::StatusOr<
|
||||
std::unique_ptr<AffineTransformation::Runner<ImageFrame, ImageFrame>>>
|
||||
CreateAffineTransformationOpenCvRunner() {
|
||||
return absl::make_unique<OpenCvRunner>();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,32 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_OPENCV_H_
|
||||
#define MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_OPENCV_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/image/affine_transformation.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
absl::StatusOr<
|
||||
std::unique_ptr<AffineTransformation::Runner<ImageFrame, ImageFrame>>>
|
||||
CreateAffineTransformationOpenCvRunner();
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_OPENCV_H_
|
||||
@@ -240,7 +240,7 @@ absl::Status BilateralFilterCalculator::RenderCpu(CalculatorContext* cc) {
|
||||
auto input_mat = mediapipe::formats::MatView(&input_frame);
|
||||
|
||||
// Only 1 or 3 channel images supported by OpenCV.
|
||||
if ((input_mat.channels() == 1 || input_mat.channels() == 3)) {
|
||||
if (!(input_mat.channels() == 1 || input_mat.channels() == 3)) {
|
||||
return absl::InternalError(
|
||||
"CPU filtering supports only 1 or 3 channel input images.");
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ using GpuBuffer = mediapipe::GpuBuffer;
|
||||
// stored on the target storage (CPU vs GPU) specified in the calculator option.
|
||||
//
|
||||
// The clone shares ownership of the input pixel data on the existing storage.
|
||||
// If the target storage is diffrent from the existing one, then the data is
|
||||
// If the target storage is different from the existing one, then the data is
|
||||
// further copied there.
|
||||
//
|
||||
// Example usage:
|
||||
|
||||
@@ -102,6 +102,10 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
|
||||
// IMAGE: ImageFrame representing the input image.
|
||||
// IMAGE_GPU: GpuBuffer representing the input image.
|
||||
//
|
||||
// OUTPUT_DIMENSIONS (optional): The output width and height in pixels as
|
||||
// pair<int, int>. If set, it will override corresponding field in calculator
|
||||
// options and input side packet.
|
||||
//
|
||||
// ROTATION_DEGREES (optional): The counterclockwise rotation angle in
|
||||
// degrees. This allows different rotation angles for different frames. It has
|
||||
// to be a multiple of 90 degrees. If provided, it overrides the
|
||||
@@ -221,6 +225,10 @@ absl::Status ImageTransformationCalculator::GetContract(
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS")) {
|
||||
cc->Inputs().Tag("OUTPUT_DIMENSIONS").Set<std::pair<int, int>>();
|
||||
}
|
||||
|
||||
if (cc->Inputs().HasTag("ROTATION_DEGREES")) {
|
||||
cc->Inputs().Tag("ROTATION_DEGREES").Set<int>();
|
||||
}
|
||||
@@ -329,6 +337,13 @@ absl::Status ImageTransformationCalculator::Process(CalculatorContext* cc) {
|
||||
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
|
||||
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
|
||||
}
|
||||
if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS") &&
|
||||
!cc->Inputs().Tag("OUTPUT_DIMENSIONS").IsEmpty()) {
|
||||
const auto& image_size =
|
||||
cc->Inputs().Tag("OUTPUT_DIMENSIONS").Get<std::pair<int, int>>();
|
||||
output_width_ = image_size.first;
|
||||
output_height_ = image_size.second;
|
||||
}
|
||||
|
||||
if (use_gpu_) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
@@ -262,6 +262,7 @@ absl::Status ScaleImageCalculator::InitializeFrameInfo(CalculatorContext* cc) {
|
||||
scale_image::FindOutputDimensions(crop_width_, crop_height_, //
|
||||
options_.target_width(), //
|
||||
options_.target_height(), //
|
||||
options_.target_max_area(), //
|
||||
options_.preserve_aspect_ratio(), //
|
||||
options_.scale_to_multiple_of(), //
|
||||
&output_width_, &output_height_));
|
||||
|
||||
@@ -28,6 +28,11 @@ message ScaleImageCalculatorOptions {
|
||||
optional int32 target_width = 1;
|
||||
optional int32 target_height = 2;
|
||||
|
||||
// If set, then automatically calculates a target_width and target_height that
|
||||
// has an area below the target max area. Aspect ratio preservation cannot be
|
||||
// disabled.
|
||||
optional int32 target_max_area = 15;
|
||||
|
||||
// If true, the image is scaled up or down proportionally so that it
|
||||
// fits inside the box represented by target_width and target_height.
|
||||
// Otherwise it is scaled to fit target_width and target_height
|
||||
|
||||
@@ -92,12 +92,21 @@ absl::Status FindOutputDimensions(int input_width, //
|
||||
int input_height, //
|
||||
int target_width, //
|
||||
int target_height, //
|
||||
int target_max_area, //
|
||||
bool preserve_aspect_ratio, //
|
||||
int scale_to_multiple_of, //
|
||||
int* output_width, int* output_height) {
|
||||
CHECK(output_width);
|
||||
CHECK(output_height);
|
||||
|
||||
if (target_max_area > 0 && input_width * input_height > target_max_area) {
|
||||
preserve_aspect_ratio = true;
|
||||
target_height = static_cast<int>(sqrt(static_cast<double>(target_max_area) /
|
||||
(static_cast<double>(input_width) /
|
||||
static_cast<double>(input_height))));
|
||||
target_width = -1; // Resize width to preserve aspect ratio.
|
||||
}
|
||||
|
||||
if (preserve_aspect_ratio) {
|
||||
RET_CHECK(scale_to_multiple_of == 2)
|
||||
<< "FindOutputDimensions always outputs width and height that are "
|
||||
@@ -164,5 +173,17 @@ absl::Status FindOutputDimensions(int input_width, //
|
||||
<< "Unable to set output dimensions based on target dimensions.";
|
||||
}
|
||||
|
||||
absl::Status FindOutputDimensions(int input_width, //
|
||||
int input_height, //
|
||||
int target_width, //
|
||||
int target_height, //
|
||||
bool preserve_aspect_ratio, //
|
||||
int scale_to_multiple_of, //
|
||||
int* output_width, int* output_height) {
|
||||
return FindOutputDimensions(
|
||||
input_width, input_height, target_width, target_height, -1,
|
||||
preserve_aspect_ratio, scale_to_multiple_of, output_width, output_height);
|
||||
}
|
||||
|
||||
} // namespace scale_image
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -34,15 +34,25 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
|
||||
int* crop_width, int* crop_height, //
|
||||
int* col_start, int* row_start);
|
||||
|
||||
// Given an input width and height, a target width and height, whether to
|
||||
// preserve the aspect ratio, and whether to round-down to the multiple of a
|
||||
// given number nearest to the targets, determine the output width and height.
|
||||
// If target_width or target_height is non-positive, then they will be set to
|
||||
// the input_width and input_height respectively. If scale_to_multiple_of is
|
||||
// less than 1, it will be treated like 1. The output_width and
|
||||
// output_height will be reduced as necessary to preserve_aspect_ratio if the
|
||||
// option is specified. If preserving the aspect ratio is desired, you must set
|
||||
// scale_to_multiple_of to 2.
|
||||
// Given an input width and height, a target width and height or max area,
|
||||
// whether to preserve the aspect ratio, and whether to round-down to the
|
||||
// multiple of a given number nearest to the targets, determine the output width
|
||||
// and height. If target_width or target_height is non-positive, then they will
|
||||
// be set to the input_width and input_height respectively. If target_area is
|
||||
// non-positive, then it will be ignored. If scale_to_multiple_of is less than
|
||||
// 1, it will be treated like 1. The output_width and output_height will be
|
||||
// reduced as necessary to preserve_aspect_ratio if the option is specified. If
|
||||
// preserving the aspect ratio is desired, you must set scale_to_multiple_of
|
||||
// to 2.
|
||||
absl::Status FindOutputDimensions(int input_width, int input_height, //
|
||||
int target_width,
|
||||
int target_height, //
|
||||
int target_max_area, //
|
||||
bool preserve_aspect_ratio, //
|
||||
int scale_to_multiple_of, //
|
||||
int* output_width, int* output_height);
|
||||
|
||||
// Backwards compatible helper.
|
||||
absl::Status FindOutputDimensions(int input_width, int input_height, //
|
||||
int target_width,
|
||||
int target_height, //
|
||||
|
||||
@@ -79,49 +79,49 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
|
||||
int output_width;
|
||||
int output_height;
|
||||
// Not scale.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, -1, true, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(200, output_width);
|
||||
EXPECT_EQ(100, output_height);
|
||||
// Not scale with odd input size.
|
||||
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, 1, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, -1, false, 1,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(201, output_width);
|
||||
EXPECT_EQ(101, output_height);
|
||||
// Scale down by 1/2.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, -1, true, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(50, output_height);
|
||||
// Scale up, doubling dimensions.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, -1, true, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(400, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Fits a 2:1 image into a 150 x 150 box. Output dimensions are always
|
||||
// visible by 2.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, -1, true, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(150, output_width);
|
||||
EXPECT_EQ(74, output_height);
|
||||
// Fits a 2:1 image into a 400 x 50 box.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, -1, true, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(50, output_height);
|
||||
// Scale to multiple number with odd targe size.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, -1, true, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(50, output_height);
|
||||
// Scale to multiple number with odd targe size.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, -1, true, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(50, output_height);
|
||||
// Scale to odd size.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, 1, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, -1, false, 1,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(151, output_width);
|
||||
EXPECT_EQ(101, output_height);
|
||||
}
|
||||
@@ -131,18 +131,18 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) {
|
||||
int output_width;
|
||||
int output_height;
|
||||
// Scale width only.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, -1, false, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(100, output_height);
|
||||
// Scale height only.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, -1, false, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(200, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Scale both dimensions.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, -1, false, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(150, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
}
|
||||
@@ -152,41 +152,78 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsDownScaleToMultipleOf) {
|
||||
int output_width;
|
||||
int output_height;
|
||||
// Set no targets, downscale to a multiple of 8.
|
||||
MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, false, 8, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, -1, false, 8,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(96, output_width);
|
||||
EXPECT_EQ(96, output_height);
|
||||
// Set width target, downscale to a multiple of 8.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 8, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, -1, false, 8,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(96, output_width);
|
||||
EXPECT_EQ(96, output_height);
|
||||
// Set height target, downscale to a multiple of 8.
|
||||
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, false, 8, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, -1, false, 8,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(200, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Set both targets, downscale to a multiple of 8.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 8, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, -1, false, 8,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(144, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Doesn't throw error if keep aspect is true and downscale multiple is 2.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, true, 2, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, -1, true, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(400, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Throws error if keep aspect is true, but downscale multiple is not 2.
|
||||
ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, true, 4, &output_width,
|
||||
&output_height),
|
||||
ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, -1, true, 4,
|
||||
&output_width, &output_height),
|
||||
testing::Not(testing::status::IsOk()));
|
||||
// Downscaling to multiple ignored if multiple is less than 2.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, false, 1, &output_width,
|
||||
&output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, -1, false, 1,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(401, output_width);
|
||||
EXPECT_EQ(201, output_height);
|
||||
}
|
||||
|
||||
// Tests scaling without keeping the aspect ratio fixed.
|
||||
TEST(ScaleImageUtilsTest, FindOutputDimensionsMaxArea) {
|
||||
int output_width;
|
||||
int output_height;
|
||||
// Smaller area.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, 9000, false, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_NEAR(
|
||||
200 / 100,
|
||||
static_cast<double>(output_width) / static_cast<double>(output_height),
|
||||
0.1f);
|
||||
EXPECT_LE(output_width * output_height, 9000);
|
||||
// Close to original area.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, 19999, false, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_NEAR(
|
||||
200.0 / 100.0,
|
||||
static_cast<double>(output_width) / static_cast<double>(output_height),
|
||||
0.1f);
|
||||
EXPECT_LE(output_width * output_height, 19999);
|
||||
// Don't scale with larger area.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, 20001, false, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(200, output_width);
|
||||
EXPECT_EQ(100, output_height);
|
||||
// Don't scale with equal area.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, 20000, false, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(200, output_width);
|
||||
EXPECT_EQ(100, output_height);
|
||||
// Don't scale at all.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, -1, false, 2,
|
||||
&output_width, &output_height));
|
||||
EXPECT_EQ(200, output_width);
|
||||
EXPECT_EQ(100, output_height);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace scale_image
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/image/warp_affine_calculator.h"
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/image/affine_transformation.h"
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/calculators/image/affine_transformation_runner_gl.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/image/affine_transformation_runner_opencv.h"
|
||||
#include "mediapipe/calculators/image/warp_affine_calculator.pb.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
AffineTransformation::BorderMode GetBorderMode(
|
||||
mediapipe::WarpAffineCalculatorOptions::BorderMode border_mode) {
|
||||
switch (border_mode) {
|
||||
case mediapipe::WarpAffineCalculatorOptions::BORDER_ZERO:
|
||||
return AffineTransformation::BorderMode::kZero;
|
||||
case mediapipe::WarpAffineCalculatorOptions::BORDER_UNSPECIFIED:
|
||||
case mediapipe::WarpAffineCalculatorOptions::BORDER_REPLICATE:
|
||||
return AffineTransformation::BorderMode::kReplicate;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ImageT>
|
||||
class WarpAffineRunnerHolder {};
|
||||
|
||||
template <>
|
||||
class WarpAffineRunnerHolder<ImageFrame> {
|
||||
public:
|
||||
using RunnerType = AffineTransformation::Runner<ImageFrame, ImageFrame>;
|
||||
absl::Status Open(CalculatorContext* cc) { return absl::OkStatus(); }
|
||||
absl::StatusOr<RunnerType*> GetRunner() {
|
||||
if (!runner_) {
|
||||
ASSIGN_OR_RETURN(runner_, CreateAffineTransformationOpenCvRunner());
|
||||
}
|
||||
return runner_.get();
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<RunnerType> runner_;
|
||||
};
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
template <>
|
||||
class WarpAffineRunnerHolder<mediapipe::GpuBuffer> {
|
||||
public:
|
||||
using RunnerType =
|
||||
AffineTransformation::Runner<mediapipe::GpuBuffer,
|
||||
std::unique_ptr<mediapipe::GpuBuffer>>;
|
||||
absl::Status Open(CalculatorContext* cc) {
|
||||
gpu_origin_ =
|
||||
cc->Options<mediapipe::WarpAffineCalculatorOptions>().gpu_origin();
|
||||
gl_helper_ = std::make_shared<mediapipe::GlCalculatorHelper>();
|
||||
return gl_helper_->Open(cc);
|
||||
}
|
||||
absl::StatusOr<RunnerType*> GetRunner() {
|
||||
if (!runner_) {
|
||||
ASSIGN_OR_RETURN(
|
||||
runner_, CreateAffineTransformationGlRunner(gl_helper_, gpu_origin_));
|
||||
}
|
||||
return runner_.get();
|
||||
}
|
||||
|
||||
private:
|
||||
mediapipe::GpuOrigin::Mode gpu_origin_;
|
||||
std::shared_ptr<mediapipe::GlCalculatorHelper> gl_helper_;
|
||||
std::unique_ptr<RunnerType> runner_;
|
||||
};
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
template <>
|
||||
class WarpAffineRunnerHolder<mediapipe::Image> {
|
||||
public:
|
||||
absl::Status Open(CalculatorContext* cc) { return runner_.Open(cc); }
|
||||
absl::StatusOr<
|
||||
AffineTransformation::Runner<mediapipe::Image, mediapipe::Image>*>
|
||||
GetRunner() {
|
||||
return &runner_;
|
||||
}
|
||||
|
||||
private:
|
||||
class Runner : public AffineTransformation::Runner<mediapipe::Image,
|
||||
mediapipe::Image> {
|
||||
public:
|
||||
absl::Status Open(CalculatorContext* cc) {
|
||||
MP_RETURN_IF_ERROR(cpu_holder_.Open(cc));
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
MP_RETURN_IF_ERROR(gpu_holder_.Open(cc));
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::StatusOr<mediapipe::Image> Run(
|
||||
const mediapipe::Image& input, const std::array<float, 16>& matrix,
|
||||
const AffineTransformation::Size& size,
|
||||
AffineTransformation::BorderMode border_mode) override {
|
||||
if (input.UsesGpu()) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
ASSIGN_OR_RETURN(auto* runner, gpu_holder_.GetRunner());
|
||||
ASSIGN_OR_RETURN(auto result, runner->Run(input.GetGpuBuffer(), matrix,
|
||||
size, border_mode));
|
||||
return mediapipe::Image(*result);
|
||||
#else
|
||||
return absl::UnavailableError("GPU support is disabled");
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
}
|
||||
ASSIGN_OR_RETURN(auto* runner, cpu_holder_.GetRunner());
|
||||
const auto& frame_ptr = input.GetImageFrameSharedPtr();
|
||||
// Wrap image into image frame.
|
||||
const ImageFrame image_frame(frame_ptr->Format(), frame_ptr->Width(),
|
||||
frame_ptr->Height(), frame_ptr->WidthStep(),
|
||||
const_cast<uint8_t*>(frame_ptr->PixelData()),
|
||||
[](uint8* data) {});
|
||||
ASSIGN_OR_RETURN(auto result,
|
||||
runner->Run(image_frame, matrix, size, border_mode));
|
||||
return mediapipe::Image(std::make_shared<ImageFrame>(std::move(result)));
|
||||
}
|
||||
|
||||
private:
|
||||
WarpAffineRunnerHolder<ImageFrame> cpu_holder_;
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
WarpAffineRunnerHolder<mediapipe::GpuBuffer> gpu_holder_;
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
};
|
||||
|
||||
Runner runner_;
|
||||
};
|
||||
|
||||
template <typename InterfaceT>
|
||||
class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
|
||||
public:
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||
if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
|
||||
std::is_same_v<InterfaceT, WarpAffineCalculator>) {
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override { return holder_.Open(cc); }
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
if (InterfaceT::kInImage(cc).IsEmpty() ||
|
||||
InterfaceT::kMatrix(cc).IsEmpty() ||
|
||||
InterfaceT::kOutputSize(cc).IsEmpty()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
const std::array<float, 16>& transform = *InterfaceT::kMatrix(cc);
|
||||
auto [out_width, out_height] = *InterfaceT::kOutputSize(cc);
|
||||
AffineTransformation::Size output_size;
|
||||
output_size.width = out_width;
|
||||
output_size.height = out_height;
|
||||
ASSIGN_OR_RETURN(auto* runner, holder_.GetRunner());
|
||||
ASSIGN_OR_RETURN(
|
||||
auto result,
|
||||
runner->Run(
|
||||
*InterfaceT::kInImage(cc), transform, output_size,
|
||||
GetBorderMode(cc->Options<mediapipe::WarpAffineCalculatorOptions>()
|
||||
.border_mode())));
|
||||
InterfaceT::kOutImage(cc).Send(std::move(result));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
WarpAffineRunnerHolder<typename decltype(InterfaceT::kInImage)::PayloadT>
|
||||
holder_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
MEDIAPIPE_NODE_IMPLEMENTATION(
|
||||
WarpAffineCalculatorImpl<WarpAffineCalculatorCpu>);
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
MEDIAPIPE_NODE_IMPLEMENTATION(
|
||||
WarpAffineCalculatorImpl<WarpAffineCalculatorGpu>);
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
MEDIAPIPE_NODE_IMPLEMENTATION(WarpAffineCalculatorImpl<WarpAffineCalculator>);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,94 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_IMAGE_WARP_AFFINE_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_IMAGE_WARP_AFFINE_CALCULATOR_H_
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/api2/port.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Runs affine transformation.
|
||||
//
|
||||
// Input:
|
||||
// IMAGE - Image/ImageFrame/GpuBuffer
|
||||
//
|
||||
// MATRIX - std::array<float, 16>
|
||||
// Used as following:
|
||||
// output(x, y) = input(matrix[0] * x + matrix[1] * y + matrix[3],
|
||||
// matrix[4] * x + matrix[5] * y + matrix[7])
|
||||
// where x and y ranges are defined by @OUTPUT_SIZE.
|
||||
//
|
||||
// OUTPUT_SIZE - std::pair<int, int>
|
||||
// Size of the output image.
|
||||
//
|
||||
// Output:
|
||||
// IMAGE - Image/ImageFrame/GpuBuffer
|
||||
//
|
||||
// Note:
|
||||
// - Output image type and format are the same as the input one.
|
||||
//
|
||||
// Usage example:
|
||||
// node {
|
||||
// calculator: "WarpAffineCalculator(Cpu|Gpu)"
|
||||
// input_stream: "IMAGE:image"
|
||||
// input_stream: "MATRIX:matrix"
|
||||
// input_stream: "OUTPUT_SIZE:size"
|
||||
// output_stream: "IMAGE:transformed_image"
|
||||
// options: {
|
||||
// [mediapipe.WarpAffineCalculatorOptions.ext] {
|
||||
// border_mode: BORDER_ZERO
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
template <typename ImageT>
|
||||
class WarpAffineCalculatorIntf : public mediapipe::api2::NodeIntf {
|
||||
public:
|
||||
static constexpr mediapipe::api2::Input<ImageT> kInImage{"IMAGE"};
|
||||
static constexpr mediapipe::api2::Input<std::array<float, 16>> kMatrix{
|
||||
"MATRIX"};
|
||||
static constexpr mediapipe::api2::Input<std::pair<int, int>> kOutputSize{
|
||||
"OUTPUT_SIZE"};
|
||||
static constexpr mediapipe::api2::Output<ImageT> kOutImage{"IMAGE"};
|
||||
};
|
||||
|
||||
class WarpAffineCalculatorCpu : public WarpAffineCalculatorIntf<ImageFrame> {
|
||||
public:
|
||||
MEDIAPIPE_NODE_INTERFACE(WarpAffineCalculatorCpu, kInImage, kMatrix,
|
||||
kOutputSize, kOutImage);
|
||||
};
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
class WarpAffineCalculatorGpu
|
||||
: public WarpAffineCalculatorIntf<mediapipe::GpuBuffer> {
|
||||
public:
|
||||
MEDIAPIPE_NODE_INTERFACE(WarpAffineCalculatorGpu, kInImage, kMatrix,
|
||||
kOutputSize, kOutImage);
|
||||
};
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
class WarpAffineCalculator : public WarpAffineCalculatorIntf<mediapipe::Image> {
|
||||
public:
|
||||
MEDIAPIPE_NODE_INTERFACE(WarpAffineCalculator, kInImage, kMatrix, kOutputSize,
|
||||
kOutImage);
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_IMAGE_WARP_AFFINE_CALCULATOR_H_
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
import "mediapipe/gpu/gpu_origin.proto";
|
||||
|
||||
message WarpAffineCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional WarpAffineCalculatorOptions ext = 373693895;
|
||||
}
|
||||
|
||||
// Pixel extrapolation methods. See @border_mode.
|
||||
enum BorderMode {
|
||||
BORDER_UNSPECIFIED = 0;
|
||||
BORDER_ZERO = 1;
|
||||
BORDER_REPLICATE = 2;
|
||||
}
|
||||
|
||||
// Pixel extrapolation method.
|
||||
// When converting image to tensor it may happen that tensor needs to read
|
||||
// pixels outside image boundaries. Border mode helps to specify how such
|
||||
// pixels will be calculated.
|
||||
//
|
||||
// BORDER_REPLICATE is used by default.
|
||||
optional BorderMode border_mode = 1;
|
||||
|
||||
// For CONVENTIONAL mode for OpenGL, input image starts at bottom and needs
|
||||
// to be flipped vertically as tensors are expected to start at top.
|
||||
// (DEFAULT or unset interpreted as CONVENTIONAL.)
|
||||
optional GpuOrigin.Mode gpu_origin = 2;
|
||||
}
|
||||
@@ -0,0 +1,615 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/image/affine_transformation.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
cv::Mat GetRgb(absl::string_view path) {
|
||||
cv::Mat bgr = cv::imread(file::JoinPath("./", path));
|
||||
cv::Mat rgb(bgr.rows, bgr.cols, CV_8UC3);
|
||||
int from_to[] = {0, 2, 1, 1, 2, 0};
|
||||
cv::mixChannels(&bgr, 1, &rgb, 1, from_to, 3);
|
||||
return rgb;
|
||||
}
|
||||
|
||||
cv::Mat GetRgba(absl::string_view path) {
|
||||
cv::Mat bgr = cv::imread(file::JoinPath("./", path));
|
||||
cv::Mat rgba(bgr.rows, bgr.cols, CV_8UC4, cv::Scalar(0, 0, 0, 0));
|
||||
int from_to[] = {0, 2, 1, 1, 2, 0};
|
||||
cv::mixChannels(&bgr, 1, &bgr, 1, from_to, 3);
|
||||
return bgr;
|
||||
}
|
||||
|
||||
// Test template.
|
||||
// No processing/assertions should be done after the function is invoked.
|
||||
void RunTest(const std::string& graph_text, const std::string& tag,
|
||||
const cv::Mat& input, cv::Mat expected_result,
|
||||
float similarity_threshold, std::array<float, 16> matrix,
|
||||
int out_width, int out_height,
|
||||
absl::optional<AffineTransformation::BorderMode> border_mode) {
|
||||
std::string border_mode_str;
|
||||
if (border_mode) {
|
||||
switch (*border_mode) {
|
||||
case AffineTransformation::BorderMode::kReplicate:
|
||||
border_mode_str = "border_mode: BORDER_REPLICATE";
|
||||
break;
|
||||
case AffineTransformation::BorderMode::kZero:
|
||||
border_mode_str = "border_mode: BORDER_ZERO";
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto graph_config = mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
absl::Substitute(graph_text, /*$0=*/border_mode_str));
|
||||
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("output_image", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
ImageFrame input_image(
|
||||
input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB,
|
||||
input.cols, input.rows, input.step, input.data, [](uint8*) {});
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_image",
|
||||
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"matrix",
|
||||
MakePacket<std::array<float, 16>>(std::move(matrix)).At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"output_size", MakePacket<std::pair<int, int>>(
|
||||
std::pair<int, int>(out_width, out_height))
|
||||
.At(Timestamp(0))));
|
||||
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
ASSERT_THAT(output_packets, testing::SizeIs(1));
|
||||
|
||||
// Get and process results.
|
||||
const ImageFrame& out_frame = output_packets[0].Get<ImageFrame>();
|
||||
cv::Mat result = formats::MatView(&out_frame);
|
||||
double similarity =
|
||||
1.0 - cv::norm(result, expected_result, cv::NORM_RELATIVE | cv::NORM_L2);
|
||||
EXPECT_GE(similarity, similarity_threshold);
|
||||
|
||||
// Fully close graph at end, otherwise calculator+tensors are destroyed
|
||||
// after calling WaitUntilDone().
|
||||
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
|
||||
MP_ASSERT_OK(graph.CloseInputStream("matrix"));
|
||||
MP_ASSERT_OK(graph.CloseInputStream("output_size"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
enum class InputType { kImageFrame, kImage };
|
||||
|
||||
// Similarity is checked against OpenCV results always, and due to differences
|
||||
// on how OpenCV and GL treats pixels there are two thresholds.
|
||||
// TODO: update to have just one threshold when OpenCV
|
||||
// implementation is updated.
|
||||
struct SimilarityConfig {
|
||||
double threshold_on_cpu;
|
||||
double threshold_on_gpu;
|
||||
};
|
||||
|
||||
void RunTest(cv::Mat input, cv::Mat expected_result,
|
||||
const SimilarityConfig& similarity, std::array<float, 16> matrix,
|
||||
int out_width, int out_height,
|
||||
absl::optional<AffineTransformation::BorderMode> border_mode) {
|
||||
RunTest(R"(
|
||||
input_stream: "input_image"
|
||||
input_stream: "output_size"
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "WarpAffineCalculatorCpu"
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_stream: "MATRIX:matrix"
|
||||
input_stream: "OUTPUT_SIZE:output_size"
|
||||
output_stream: "IMAGE:output_image"
|
||||
options {
|
||||
[mediapipe.WarpAffineCalculatorOptions.ext] {
|
||||
$0 # border mode
|
||||
}
|
||||
}
|
||||
}
|
||||
)",
|
||||
"cpu", input, expected_result, similarity.threshold_on_cpu, matrix,
|
||||
out_width, out_height, border_mode);
|
||||
|
||||
RunTest(R"(
|
||||
input_stream: "input_image"
|
||||
input_stream: "output_size"
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "ToImageCalculator"
|
||||
input_stream: "IMAGE_CPU:input_image"
|
||||
output_stream: "IMAGE:input_image_unified"
|
||||
}
|
||||
node {
|
||||
calculator: "WarpAffineCalculator"
|
||||
input_stream: "IMAGE:input_image_unified"
|
||||
input_stream: "MATRIX:matrix"
|
||||
input_stream: "OUTPUT_SIZE:output_size"
|
||||
output_stream: "IMAGE:output_image_unified"
|
||||
options {
|
||||
[mediapipe.WarpAffineCalculatorOptions.ext] {
|
||||
$0 # border mode
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "FromImageCalculator"
|
||||
input_stream: "IMAGE:output_image_unified"
|
||||
output_stream: "IMAGE_CPU:output_image"
|
||||
}
|
||||
)",
|
||||
"cpu_image", input, expected_result, similarity.threshold_on_cpu,
|
||||
matrix, out_width, out_height, border_mode);
|
||||
|
||||
RunTest(R"(
|
||||
input_stream: "input_image"
|
||||
input_stream: "output_size"
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "ImageFrameToGpuBufferCalculator"
|
||||
input_stream: "input_image"
|
||||
output_stream: "input_image_gpu"
|
||||
}
|
||||
node {
|
||||
calculator: "WarpAffineCalculatorGpu"
|
||||
input_stream: "IMAGE:input_image_gpu"
|
||||
input_stream: "MATRIX:matrix"
|
||||
input_stream: "OUTPUT_SIZE:output_size"
|
||||
output_stream: "IMAGE:output_image_gpu"
|
||||
options {
|
||||
[mediapipe.WarpAffineCalculatorOptions.ext] {
|
||||
$0 # border mode
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "GpuBufferToImageFrameCalculator"
|
||||
input_stream: "output_image_gpu"
|
||||
output_stream: "output_image"
|
||||
}
|
||||
)",
|
||||
"gpu", input, expected_result, similarity.threshold_on_gpu, matrix,
|
||||
out_width, out_height, border_mode);
|
||||
|
||||
RunTest(R"(
|
||||
input_stream: "input_image"
|
||||
input_stream: "output_size"
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "ImageFrameToGpuBufferCalculator"
|
||||
input_stream: "input_image"
|
||||
output_stream: "input_image_gpu"
|
||||
}
|
||||
node {
|
||||
calculator: "ToImageCalculator"
|
||||
input_stream: "IMAGE_GPU:input_image_gpu"
|
||||
output_stream: "IMAGE:input_image_unified"
|
||||
}
|
||||
node {
|
||||
calculator: "WarpAffineCalculator"
|
||||
input_stream: "IMAGE:input_image_unified"
|
||||
input_stream: "MATRIX:matrix"
|
||||
input_stream: "OUTPUT_SIZE:output_size"
|
||||
output_stream: "IMAGE:output_image_unified"
|
||||
options {
|
||||
[mediapipe.WarpAffineCalculatorOptions.ext] {
|
||||
$0 # border mode
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "FromImageCalculator"
|
||||
input_stream: "IMAGE:output_image_unified"
|
||||
output_stream: "IMAGE_GPU:output_image_gpu"
|
||||
}
|
||||
node {
|
||||
calculator: "GpuBufferToImageFrameCalculator"
|
||||
input_stream: "output_image_gpu"
|
||||
output_stream: "output_image"
|
||||
}
|
||||
)",
|
||||
"gpu_image", input, expected_result, similarity.threshold_on_gpu,
|
||||
matrix, out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
|
||||
bool keep_aspect_ratio, int out_width,
|
||||
int out_height) {
|
||||
std::array<float, 16> transform_mat;
|
||||
mediapipe::RotatedRect roi_absolute =
|
||||
mediapipe::GetRoi(input.cols, input.rows, roi);
|
||||
mediapipe::PadRoi(out_width, out_height, keep_aspect_ratio, &roi_absolute)
|
||||
.IgnoreError();
|
||||
mediapipe::GetRotatedSubRectToRectTransformMatrix(
|
||||
roi_absolute, input.cols, input.rows,
|
||||
/*flip_horizontaly=*/false, &transform_mat);
|
||||
return transform_mat;
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspect) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.65f);
|
||||
roi.set_y_center(0.4f);
|
||||
roi.set_width(0.5f);
|
||||
roi.set_height(0.5f);
|
||||
roi.set_rotation(0);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png");
|
||||
int out_width = 256;
|
||||
int out_height = 256;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode = {};
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.82},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectBorderZero) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.65f);
|
||||
roi.set_y_center(0.4f);
|
||||
roi.set_width(0.5f);
|
||||
roi.set_height(0.5f);
|
||||
roi.set_rotation(0);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/"
|
||||
"medium_sub_rect_keep_aspect_border_zero.png");
|
||||
int out_width = 256;
|
||||
int out_height = 256;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kZero;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.81},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectWithRotation) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.65f);
|
||||
roi.set_y_center(0.4f);
|
||||
roi.set_width(0.5f);
|
||||
roi.set_height(0.5f);
|
||||
roi.set_rotation(M_PI * 90.0f / 180.0f);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/"
|
||||
"medium_sub_rect_keep_aspect_with_rotation.png");
|
||||
int out_width = 256;
|
||||
int out_height = 256;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kReplicate;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.77},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectWithRotationBorderZero) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.65f);
|
||||
roi.set_y_center(0.4f);
|
||||
roi.set_width(0.5f);
|
||||
roi.set_height(0.5f);
|
||||
roi.set_rotation(M_PI * 90.0f / 180.0f);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/"
|
||||
"medium_sub_rect_keep_aspect_with_rotation_border_zero.png");
|
||||
int out_width = 256;
|
||||
int out_height = 256;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kZero;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.75},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, MediumSubRectWithRotation) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.65f);
|
||||
roi.set_y_center(0.4f);
|
||||
roi.set_width(0.5f);
|
||||
roi.set_height(0.5f);
|
||||
roi.set_rotation(M_PI * -45.0f / 180.0f);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png");
|
||||
int out_width = 256;
|
||||
int out_height = 256;
|
||||
bool keep_aspect_ratio = false;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kReplicate;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.81},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, MediumSubRectWithRotationBorderZero) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.65f);
|
||||
roi.set_y_center(0.4f);
|
||||
roi.set_width(0.5f);
|
||||
roi.set_height(0.5f);
|
||||
roi.set_rotation(M_PI * -45.0f / 180.0f);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/"
|
||||
"medium_sub_rect_with_rotation_border_zero.png");
|
||||
int out_width = 256;
|
||||
int out_height = 256;
|
||||
bool keep_aspect_ratio = false;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kZero;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.80},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, LargeSubRect) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.5f);
|
||||
roi.set_height(1.1f);
|
||||
roi.set_rotation(0);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/large_sub_rect.png");
|
||||
int out_width = 128;
|
||||
int out_height = 128;
|
||||
bool keep_aspect_ratio = false;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kReplicate;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.95},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, LargeSubRectBorderZero) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.5f);
|
||||
roi.set_height(1.1f);
|
||||
roi.set_rotation(0);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png");
|
||||
int out_width = 128;
|
||||
int out_height = 128;
|
||||
bool keep_aspect_ratio = false;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kZero;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.92},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspect) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.5f);
|
||||
roi.set_height(1.1f);
|
||||
roi.set_rotation(0);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png");
|
||||
int out_width = 128;
|
||||
int out_height = 128;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kReplicate;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.97},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectBorderZero) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.5f);
|
||||
roi.set_height(1.1f);
|
||||
roi.set_rotation(0);
|
||||
auto input = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgb(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/"
|
||||
"large_sub_rect_keep_aspect_border_zero.png");
|
||||
int out_width = 128;
|
||||
int out_height = 128;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kZero;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.97},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectWithRotation) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.5f);
|
||||
roi.set_height(1.1f);
|
||||
roi.set_rotation(M_PI * -15.0f / 180.0f);
|
||||
auto input = GetRgba(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgba(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/"
|
||||
"large_sub_rect_keep_aspect_with_rotation.png");
|
||||
int out_width = 128;
|
||||
int out_height = 128;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode = {};
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.91},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectWithRotationBorderZero) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.5f);
|
||||
roi.set_height(1.1f);
|
||||
roi.set_rotation(M_PI * -15.0f / 180.0f);
|
||||
auto input = GetRgba(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgba(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/"
|
||||
"large_sub_rect_keep_aspect_with_rotation_border_zero.png");
|
||||
int out_width = 128;
|
||||
int out_height = 128;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kZero;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.88},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, NoOp) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.0f);
|
||||
roi.set_height(1.0f);
|
||||
roi.set_rotation(0);
|
||||
auto input = GetRgba(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgba(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/noop_except_range.png");
|
||||
int out_width = 64;
|
||||
int out_height = 128;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kReplicate;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.99},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
TEST(WarpAffineCalculatorTest, NoOpBorderZero) {
|
||||
mediapipe::NormalizedRect roi;
|
||||
roi.set_x_center(0.5f);
|
||||
roi.set_y_center(0.5f);
|
||||
roi.set_width(1.0f);
|
||||
roi.set_height(1.0f);
|
||||
roi.set_rotation(0);
|
||||
auto input = GetRgba(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/input.jpg");
|
||||
auto expected_output = GetRgba(
|
||||
"/mediapipe/calculators/"
|
||||
"tensor/testdata/image_to_tensor/noop_except_range.png");
|
||||
int out_width = 64;
|
||||
int out_height = 128;
|
||||
bool keep_aspect_ratio = true;
|
||||
std::optional<AffineTransformation::BorderMode> border_mode =
|
||||
AffineTransformation::BorderMode::kZero;
|
||||
RunTest(input, expected_output,
|
||||
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.99},
|
||||
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
|
||||
out_width, out_height, border_mode);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -26,6 +26,11 @@ licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
exports_files(
|
||||
glob(["testdata/image_to_tensor/*"]),
|
||||
visibility = ["//mediapipe/calculators/image:__subpackages__"],
|
||||
)
|
||||
|
||||
selects.config_setting_group(
|
||||
name = "compute_shader_unavailable",
|
||||
match_any = [
|
||||
|
||||
@@ -87,9 +87,9 @@ using GpuBuffer = mediapipe::GpuBuffer;
|
||||
// TENSORS - std::vector<Tensor>
|
||||
// Vector containing a single Tensor populated with an extrated RGB image.
|
||||
// MATRIX - std::array<float, 16> @Optional
|
||||
// An std::array<float, 16> representing a 4x4 row-major-order matrix which
|
||||
// can be used to map a point on the output tensor to a point on the input
|
||||
// image.
|
||||
// An std::array<float, 16> representing a 4x4 row-major-order matrix that
|
||||
// maps a point on the input image to a point on the output tensor, and
|
||||
// can be used to reverse the mapping by inverting the matrix.
|
||||
// LETTERBOX_PADDING - std::array<float, 4> @Optional
|
||||
// An std::array<float, 4> representing the letterbox padding from the 4
|
||||
// sides ([left, top, right, bottom]) of the output image, normalized to
|
||||
|
||||
@@ -33,7 +33,7 @@ class InferenceCalculatorSelectorImpl
|
||||
absl::StatusOr<CalculatorGraphConfig> GetConfig(
|
||||
const CalculatorGraphConfig::Node& subgraph_node) {
|
||||
const auto& options =
|
||||
Subgraph::GetOptions<::mediapipe::InferenceCalculatorOptions>(
|
||||
Subgraph::GetOptions<mediapipe::InferenceCalculatorOptions>(
|
||||
subgraph_node);
|
||||
std::vector<absl::string_view> impls;
|
||||
const bool should_use_gpu =
|
||||
|
||||
@@ -99,8 +99,13 @@ class InferenceCalculator : public NodeIntf {
|
||||
kSideInCustomOpResolver{"CUSTOM_OP_RESOLVER"};
|
||||
static constexpr SideInput<TfLiteModelPtr>::Optional kSideInModel{"MODEL"};
|
||||
static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
|
||||
static constexpr SideInput<std::string>::Optional kNnApiDelegateCacheDir{
|
||||
"NNAPI_CACHE_DIR"};
|
||||
static constexpr SideInput<std::string>::Optional kNnApiDelegateModelToken{
|
||||
"NNAPI_MODEL_TOKEN"};
|
||||
MEDIAPIPE_NODE_CONTRACT(kInTensors, kSideInCustomOpResolver, kSideInModel,
|
||||
kOutTensors);
|
||||
kOutTensors, kNnApiDelegateCacheDir,
|
||||
kNnApiDelegateModelToken);
|
||||
|
||||
protected:
|
||||
using TfLiteDelegatePtr =
|
||||
|
||||
@@ -67,9 +67,32 @@ message InferenceCalculatorOptions {
|
||||
// Only available for OpenCL delegate on Android.
|
||||
// Kernel caching will only be enabled if this path is set.
|
||||
optional string cached_kernel_path = 2;
|
||||
|
||||
// Encapsulated compilation/runtime tradeoffs.
|
||||
enum InferenceUsage {
|
||||
UNSPECIFIED = 0;
|
||||
|
||||
// InferenceRunner will be used only once. Therefore, it is important to
|
||||
// minimize bootstrap time as well.
|
||||
FAST_SINGLE_ANSWER = 1;
|
||||
|
||||
// Prefer maximizing the throughput. Same inference runner will be used
|
||||
// repeatedly on different inputs.
|
||||
SUSTAINED_SPEED = 2;
|
||||
}
|
||||
optional InferenceUsage usage = 5 [default = SUSTAINED_SPEED];
|
||||
}
|
||||
|
||||
// Android only.
|
||||
message Nnapi {}
|
||||
message Nnapi {
|
||||
// Directory to store compilation cache. If unspecified, NNAPI will not
|
||||
// try caching the compilation.
|
||||
optional string cache_dir = 1;
|
||||
// Unique token identifying the model. It is the caller's responsibility
|
||||
// to ensure there is no clash of the tokens. If unspecified, NNAPI will
|
||||
// not try caching the compilation.
|
||||
optional string model_token = 2;
|
||||
}
|
||||
message Xnnpack {
|
||||
// Number of threads for XNNPACK delegate. (By default, calculator tries
|
||||
// to choose optimal number of threads depending on the device.)
|
||||
|
||||
@@ -181,9 +181,21 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegate(CalculatorContext* cc) {
|
||||
// Attempt to use NNAPI.
|
||||
// If not supported, the default CPU delegate will be created and used.
|
||||
interpreter_->SetAllowFp16PrecisionForFp32(1);
|
||||
delegate_ = TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
|
||||
// No need to free according to tflite::NnApiDelegate() documentation.
|
||||
});
|
||||
tflite::StatefulNnApiDelegate::Options options;
|
||||
const auto& nnapi = calculator_opts.delegate().nnapi();
|
||||
// Set up cache_dir and model_token for NNAPI compilation cache.
|
||||
options.cache_dir =
|
||||
nnapi.has_cache_dir() ? nnapi.cache_dir().c_str() : nullptr;
|
||||
if (!kNnApiDelegateCacheDir(cc).IsEmpty()) {
|
||||
options.cache_dir = kNnApiDelegateCacheDir(cc).Get().c_str();
|
||||
}
|
||||
options.model_token =
|
||||
nnapi.has_model_token() ? nnapi.model_token().c_str() : nullptr;
|
||||
if (!kNnApiDelegateModelToken(cc).IsEmpty()) {
|
||||
options.model_token = kNnApiDelegateModelToken(cc).Get().c_str();
|
||||
}
|
||||
delegate_ = TfLiteDelegatePtr(new tflite::StatefulNnApiDelegate(options),
|
||||
[](TfLiteDelegate*) {});
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
|
||||
kTfLiteOk);
|
||||
return absl::OkStatus();
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/calculators/tensor/inference_calculator.h"
|
||||
#include "mediapipe/util/tflite/config.h"
|
||||
|
||||
@@ -65,6 +66,8 @@ class InferenceCalculatorGlImpl
|
||||
bool allow_precision_loss_ = false;
|
||||
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
|
||||
tflite_gpu_runner_api_;
|
||||
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::InferenceUsage
|
||||
tflite_gpu_runner_usage_;
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
|
||||
#if MEDIAPIPE_TFLITE_GPU_SUPPORTED
|
||||
@@ -96,6 +99,7 @@ absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
|
||||
options.delegate().gpu().use_advanced_gpu_api();
|
||||
allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
|
||||
tflite_gpu_runner_api_ = options.delegate().gpu().api();
|
||||
tflite_gpu_runner_usage_ = options.delegate().gpu().usage();
|
||||
use_kernel_caching_ = use_advanced_gpu_api_ &&
|
||||
options.delegate().gpu().has_cached_kernel_path();
|
||||
use_gpu_delegate_ = !use_advanced_gpu_api_;
|
||||
@@ -253,9 +257,27 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
|
||||
: tflite::gpu::InferencePriority::MAX_PRECISION;
|
||||
options.priority2 = tflite::gpu::InferencePriority::AUTO;
|
||||
options.priority3 = tflite::gpu::InferencePriority::AUTO;
|
||||
options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
|
||||
switch (tflite_gpu_runner_usage_) {
|
||||
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::
|
||||
FAST_SINGLE_ANSWER: {
|
||||
options.usage = tflite::gpu::InferenceUsage::FAST_SINGLE_ANSWER;
|
||||
break;
|
||||
}
|
||||
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::
|
||||
SUSTAINED_SPEED: {
|
||||
options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
|
||||
break;
|
||||
}
|
||||
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::UNSPECIFIED: {
|
||||
return absl::InternalError("inference usage need to be specified.");
|
||||
}
|
||||
}
|
||||
tflite_gpu_runner_ = std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
|
||||
switch (tflite_gpu_runner_api_) {
|
||||
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: {
|
||||
// Do not need to force any specific API.
|
||||
break;
|
||||
}
|
||||
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::OPENGL: {
|
||||
tflite_gpu_runner_->ForceOpenGL();
|
||||
break;
|
||||
@@ -264,10 +286,6 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
|
||||
tflite_gpu_runner_->ForceOpenCL();
|
||||
break;
|
||||
}
|
||||
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: {
|
||||
// Do not need to force any specific API.
|
||||
break;
|
||||
}
|
||||
}
|
||||
MP_RETURN_IF_ERROR(tflite_gpu_runner_->InitializeWithModel(
|
||||
model, op_resolver, /*allow_quant_ops=*/true));
|
||||
|
||||
@@ -517,8 +517,8 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
|
||||
uniform sampler2D frame;
|
||||
|
||||
void main() {
|
||||
$1 // flip
|
||||
vec4 pixel = texture2D(frame, sample_coordinate);
|
||||
vec2 coord = $1
|
||||
vec4 pixel = texture2D(frame, coord);
|
||||
$2 // normalize [-1,1]
|
||||
fragColor.r = pixel.r; // r channel
|
||||
$3 // g & b channels
|
||||
@@ -526,8 +526,9 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
|
||||
})",
|
||||
/*$0=*/single_channel ? "vec1" : "vec4",
|
||||
/*$1=*/
|
||||
flip_vertically_ ? "sample_coordinate.y = 1.0 - sample_coordinate.y;"
|
||||
: "",
|
||||
flip_vertically_
|
||||
? "vec2(sample_coordinate.x, 1.0 - sample_coordinate.y);"
|
||||
: "sample_coordinate;",
|
||||
/*$2=*/output_range_.has_value()
|
||||
? absl::Substitute("pixel = pixel * float($0) + float($1);",
|
||||
(output_range_->second - output_range_->first),
|
||||
|
||||
@@ -88,6 +88,13 @@ proto_library(
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "tensor_to_vector_string_calculator_options_proto",
|
||||
srcs = ["tensor_to_vector_string_calculator_options.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "unpack_media_sequence_calculator_proto",
|
||||
srcs = ["unpack_media_sequence_calculator.proto"],
|
||||
@@ -257,6 +264,14 @@ mediapipe_cc_proto_library(
|
||||
deps = [":tensor_to_vector_float_calculator_options_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "tensor_to_vector_string_calculator_options_cc_proto",
|
||||
srcs = ["tensor_to_vector_string_calculator_options.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":tensor_to_vector_string_calculator_options_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "unpack_media_sequence_calculator_cc_proto",
|
||||
srcs = ["unpack_media_sequence_calculator.proto"],
|
||||
@@ -572,9 +587,21 @@ cc_library(
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
] + select({
|
||||
"//conditions:default": [
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
],
|
||||
}),
|
||||
"//mediapipe:android": [],
|
||||
}) + select(
|
||||
{
|
||||
"//conditions:default": [
|
||||
],
|
||||
},
|
||||
) + select(
|
||||
{
|
||||
"//conditions:default": [
|
||||
],
|
||||
"//mediapipe:android": [
|
||||
],
|
||||
},
|
||||
),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
@@ -694,6 +721,26 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensor_to_vector_string_calculator",
|
||||
srcs = ["tensor_to_vector_string_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
":tensor_to_vector_string_calculator_options_cc_proto",
|
||||
] + select({
|
||||
"//conditions:default": [
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
],
|
||||
"//mediapipe:android": [
|
||||
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "unpack_media_sequence_calculator",
|
||||
srcs = ["unpack_media_sequence_calculator.cc"],
|
||||
@@ -864,6 +911,7 @@ cc_test(
|
||||
"//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
@@ -1058,6 +1106,20 @@ cc_test(
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensor_to_vector_string_calculator_test",
|
||||
srcs = ["tensor_to_vector_string_calculator_test.cc"],
|
||||
deps = [
|
||||
":tensor_to_vector_string_calculator",
|
||||
":tensor_to_vector_string_calculator_options_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "unpack_media_sequence_calculator_test",
|
||||
srcs = ["unpack_media_sequence_calculator_test.cc"],
|
||||
|
||||
@@ -37,6 +37,7 @@ const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
|
||||
const char kImageTag[] = "IMAGE";
|
||||
const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_";
|
||||
const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_";
|
||||
const char kBytesFeaturePrefixTag[] = "BYTES_FEATURE_";
|
||||
const char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
|
||||
const char kBBoxTag[] = "BBOX";
|
||||
const char kKeypointsTag[] = "KEYPOINTS";
|
||||
@@ -153,6 +154,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
|
||||
cc->Inputs().Tag(tag).Set<std::vector<float>>();
|
||||
}
|
||||
if (absl::StartsWith(tag, kBytesFeaturePrefixTag)) {
|
||||
cc->Inputs().Tag(tag).Set<std::vector<std::string>>();
|
||||
}
|
||||
}
|
||||
|
||||
CHECK(cc->Outputs().HasTag(kSequenceExampleTag) ||
|
||||
@@ -231,6 +235,13 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
mpms::ClearFeatureFloats(key, sequence_.get());
|
||||
mpms::ClearFeatureTimestamp(key, sequence_.get());
|
||||
}
|
||||
if (absl::StartsWith(tag, kBytesFeaturePrefixTag)) {
|
||||
std::string key = tag.substr(sizeof(kBytesFeaturePrefixTag) /
|
||||
sizeof(*kBytesFeaturePrefixTag) -
|
||||
1);
|
||||
mpms::ClearFeatureBytes(key, sequence_.get());
|
||||
mpms::ClearFeatureTimestamp(key, sequence_.get());
|
||||
}
|
||||
if (absl::StartsWith(tag, kKeypointsTag)) {
|
||||
std::string key =
|
||||
tag.substr(sizeof(kKeypointsTag) / sizeof(*kKeypointsTag) - 1);
|
||||
@@ -243,11 +254,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
}
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag(kSequenceExampleTag)) {
|
||||
cc->Outputs()
|
||||
.Tag(kSequenceExampleTag)
|
||||
.SetNextTimestampBound(Timestamp::Max());
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -305,7 +311,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
if (cc->Outputs().HasTag(kSequenceExampleTag)) {
|
||||
cc->Outputs()
|
||||
.Tag(kSequenceExampleTag)
|
||||
.Add(sequence_.release(), Timestamp::PostStream());
|
||||
.Add(sequence_.release(), options.output_as_zero_timestamp()
|
||||
? Timestamp(0ll)
|
||||
: Timestamp::PostStream());
|
||||
}
|
||||
sequence_.reset();
|
||||
|
||||
@@ -408,6 +416,17 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
cc->Inputs().Tag(tag).Get<std::vector<float>>(),
|
||||
sequence_.get());
|
||||
}
|
||||
if (absl::StartsWith(tag, kBytesFeaturePrefixTag) &&
|
||||
!cc->Inputs().Tag(tag).IsEmpty()) {
|
||||
std::string key = tag.substr(sizeof(kBytesFeaturePrefixTag) /
|
||||
sizeof(*kBytesFeaturePrefixTag) -
|
||||
1);
|
||||
mpms::AddFeatureTimestamp(key, cc->InputTimestamp().Value(),
|
||||
sequence_.get());
|
||||
mpms::AddFeatureBytes(
|
||||
key, cc->Inputs().Tag(tag).Get<std::vector<std::string>>(),
|
||||
sequence_.get());
|
||||
}
|
||||
if (absl::StartsWith(tag, kBBoxTag) && !cc->Inputs().Tag(tag).IsEmpty()) {
|
||||
std::string key = "";
|
||||
if (tag != kBBoxTag) {
|
||||
|
||||
@@ -65,4 +65,7 @@ message PackMediaSequenceCalculatorOptions {
|
||||
// If true, will return an error status if an output sequence would be too
|
||||
// many bytes to serialize.
|
||||
optional bool skip_large_sequences = 7 [default = true];
|
||||
|
||||
// If true/false, outputs the SequenceExample at timestamp 0/PostStream.
|
||||
optional bool output_as_zero_timestamp = 8 [default = false];
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/util/sequence/media_sequence.h"
|
||||
#include "tensorflow/core/example/example.pb.h"
|
||||
#include "tensorflow/core/example/feature.pb.h"
|
||||
@@ -39,12 +40,33 @@ namespace {
|
||||
namespace tf = ::tensorflow;
|
||||
namespace mpms = mediapipe::mediasequence;
|
||||
|
||||
constexpr char kBboxTag[] = "BBOX";
|
||||
constexpr char kEncodedMediaStartTimestampTag[] =
|
||||
"ENCODED_MEDIA_START_TIMESTAMP";
|
||||
constexpr char kEncodedMediaTag[] = "ENCODED_MEDIA";
|
||||
constexpr char kClassSegmentationTag[] = "CLASS_SEGMENTATION";
|
||||
constexpr char kKeypointsTestTag[] = "KEYPOINTS_TEST";
|
||||
constexpr char kBboxPredictedTag[] = "BBOX_PREDICTED";
|
||||
constexpr char kAudioOtherTag[] = "AUDIO_OTHER";
|
||||
constexpr char kAudioTestTag[] = "AUDIO_TEST";
|
||||
constexpr char kBytesFeatureOtherTag[] = "BYTES_FEATURE_OTHER";
|
||||
constexpr char kBytesFeatureTestTag[] = "BYTES_FEATURE_TEST";
|
||||
constexpr char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
|
||||
constexpr char kFloatContextFeatureOtherTag[] = "FLOAT_CONTEXT_FEATURE_OTHER";
|
||||
constexpr char kFloatContextFeatureTestTag[] = "FLOAT_CONTEXT_FEATURE_TEST";
|
||||
constexpr char kFloatFeatureOtherTag[] = "FLOAT_FEATURE_OTHER";
|
||||
constexpr char kFloatFeatureTestTag[] = "FLOAT_FEATURE_TEST";
|
||||
constexpr char kImagePrefixTag[] = "IMAGE_PREFIX";
|
||||
constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
|
||||
constexpr char kImageTag[] = "IMAGE";
|
||||
|
||||
class PackMediaSequenceCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUpCalculator(const std::vector<std::string>& input_streams,
|
||||
const tf::Features& features,
|
||||
bool output_only_if_all_present,
|
||||
bool replace_instead_of_append) {
|
||||
const bool output_only_if_all_present,
|
||||
const bool replace_instead_of_append,
|
||||
const bool output_as_zero_timestamp = false) {
|
||||
CalculatorGraphConfig::Node config;
|
||||
config.set_calculator("PackMediaSequenceCalculator");
|
||||
config.add_input_side_packet("SEQUENCE_EXAMPLE:input_sequence");
|
||||
@@ -57,6 +79,7 @@ class PackMediaSequenceCalculatorTest : public ::testing::Test {
|
||||
*options->mutable_context_feature_map() = features;
|
||||
options->set_output_only_if_all_present(output_only_if_all_present);
|
||||
options->set_replace_data_instead_of_append(replace_instead_of_append);
|
||||
options->set_output_as_zero_timestamp(output_as_zero_timestamp);
|
||||
runner_ = ::absl::make_unique<CalculatorRunner>(config);
|
||||
}
|
||||
|
||||
@@ -80,17 +103,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImages) {
|
||||
for (int i = 0; i < num_images; ++i) {
|
||||
auto image_ptr =
|
||||
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
|
||||
runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
|
||||
runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
|
||||
Adopt(image_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -124,17 +147,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoPrefixedImages) {
|
||||
auto image_ptr =
|
||||
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
|
||||
runner_->MutableInputs()
|
||||
->Tag("IMAGE_PREFIX")
|
||||
->Tag(kImagePrefixTag)
|
||||
.packets.push_back(Adopt(image_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -158,21 +181,21 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) {
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FLOAT_FEATURE_TEST")
|
||||
->Tag(kFloatFeatureTestTag)
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
|
||||
vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FLOAT_FEATURE_OTHER")
|
||||
->Tag(kFloatFeatureOtherTag)
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -194,20 +217,65 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) {
|
||||
SetUpCalculator(
|
||||
{"FLOAT_CONTEXT_FEATURE_TEST:test", "FLOAT_CONTEXT_FEATURE_OTHER:test2"},
|
||||
{}, false, true);
|
||||
auto input_sequence = absl::make_unique<tf::SequenceExample>();
|
||||
TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBytesLists) {
|
||||
SetUpCalculator({"BYTES_FEATURE_TEST:test", "BYTES_FEATURE_OTHER:test2"}, {},
|
||||
false, true);
|
||||
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
|
||||
|
||||
auto vf_ptr = absl::make_unique<std::vector<float>>(2, 3);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FLOAT_CONTEXT_FEATURE_TEST")
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
|
||||
vf_ptr = absl::make_unique<std::vector<float>>(2, 4);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FLOAT_CONTEXT_FEATURE_OTHER")
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
|
||||
int num_timesteps = 2;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto vs_ptr = ::absl::make_unique<std::vector<std::string>>(
|
||||
2, absl::StrCat("foo", 2 << i));
|
||||
runner_->MutableInputs()
|
||||
->Tag(kBytesFeatureTestTag)
|
||||
.packets.push_back(Adopt(vs_ptr.release()).At(Timestamp(i)));
|
||||
vs_ptr = ::absl::make_unique<std::vector<std::string>>(
|
||||
2, absl::StrCat("bar", 2 << i));
|
||||
runner_->MutableInputs()
|
||||
->Tag(kBytesFeatureOtherTag)
|
||||
.packets.push_back(Adopt(vs_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
|
||||
ASSERT_EQ(num_timesteps,
|
||||
mpms::GetFeatureTimestampSize("TEST", output_sequence));
|
||||
ASSERT_EQ(num_timesteps, mpms::GetFeatureBytesSize("TEST", output_sequence));
|
||||
ASSERT_EQ(num_timesteps,
|
||||
mpms::GetFeatureTimestampSize("OTHER", output_sequence));
|
||||
ASSERT_EQ(num_timesteps, mpms::GetFeatureBytesSize("OTHER", output_sequence));
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
ASSERT_EQ(i, mpms::GetFeatureTimestampAt("TEST", output_sequence, i));
|
||||
ASSERT_THAT(mpms::GetFeatureBytesAt("TEST", output_sequence, i),
|
||||
::testing::ElementsAreArray(
|
||||
std::vector<std::string>(2, absl::StrCat("foo", 2 << i))));
|
||||
ASSERT_EQ(i, mpms::GetFeatureTimestampAt("OTHER", output_sequence, i));
|
||||
ASSERT_THAT(mpms::GetFeatureBytesAt("OTHER", output_sequence, i),
|
||||
::testing::ElementsAreArray(
|
||||
std::vector<std::string>(2, absl::StrCat("bar", 2 << i))));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(PackMediaSequenceCalculatorTest, OutputAsZeroTimestamp) {
|
||||
SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true, true);
|
||||
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
|
||||
|
||||
int num_timesteps = 2;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FLOAT_FEATURE_TEST")
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
Adopt(input_sequence.release());
|
||||
@@ -217,6 +285,32 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) {
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(output_packets[0].Timestamp().Value(), 0ll);
|
||||
}
|
||||
|
||||
TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) {
|
||||
SetUpCalculator(
|
||||
{"FLOAT_CONTEXT_FEATURE_TEST:test", "FLOAT_CONTEXT_FEATURE_OTHER:test2"},
|
||||
{}, false, true);
|
||||
auto input_sequence = absl::make_unique<tf::SequenceExample>();
|
||||
|
||||
auto vf_ptr = absl::make_unique<std::vector<float>>(2, 3);
|
||||
runner_->MutableInputs()
|
||||
->Tag(kFloatContextFeatureTestTag)
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
|
||||
vf_ptr = absl::make_unique<std::vector<float>>(2, 4);
|
||||
runner_->MutableInputs()
|
||||
->Tag(kFloatContextFeatureOtherTag)
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
|
||||
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
|
||||
@@ -233,7 +327,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
|
||||
SetUpCalculator({"IMAGE:images"}, context, false, true);
|
||||
|
||||
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
@@ -242,13 +336,13 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
|
||||
encoded_image.set_encoded_image(bytes.data(), bytes.size());
|
||||
auto image_ptr =
|
||||
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
|
||||
runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
|
||||
runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
|
||||
Adopt(image_ptr.release()).At(Timestamp(0)));
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -281,17 +375,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoForwardFlowEncodeds) {
|
||||
auto flow_ptr =
|
||||
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_flow);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FORWARD_FLOW_ENCODED")
|
||||
->Tag(kForwardFlowEncodedTag)
|
||||
.packets.push_back(Adopt(flow_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -345,17 +439,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBBoxDetections) {
|
||||
detections->push_back(detection);
|
||||
|
||||
runner_->MutableInputs()
|
||||
->Tag("BBOX_PREDICTED")
|
||||
->Tag(kBboxPredictedTag)
|
||||
.packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -424,11 +518,11 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithoutImageDims) {
|
||||
detections->push_back(detection);
|
||||
|
||||
runner_->MutableInputs()
|
||||
->Tag("BBOX_PREDICTED")
|
||||
->Tag(kBboxPredictedTag)
|
||||
.packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
auto status = runner_->Run();
|
||||
@@ -472,7 +566,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
|
||||
detections->push_back(detection);
|
||||
|
||||
runner_->MutableInputs()
|
||||
->Tag("BBOX_PREDICTED")
|
||||
->Tag(kBboxPredictedTag)
|
||||
.packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
|
||||
}
|
||||
cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
@@ -487,16 +581,16 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
|
||||
for (int i = 0; i < num_images; ++i) {
|
||||
auto image_ptr =
|
||||
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
|
||||
runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
|
||||
runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
|
||||
Adopt(image_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -538,18 +632,18 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoKeypoints) {
|
||||
absl::flat_hash_map<std::string, std::vector<std::pair<float, float>>>
|
||||
points = {{"HEAD", {{0.1, 0.2}, {0.3, 0.4}}}, {"TAIL", {{0.5, 0.6}}}};
|
||||
runner_->MutableInputs()
|
||||
->Tag("KEYPOINTS_TEST")
|
||||
->Tag(kKeypointsTestTag)
|
||||
.packets.push_back(PointToForeign(&points).At(Timestamp(0)));
|
||||
runner_->MutableInputs()
|
||||
->Tag("KEYPOINTS_TEST")
|
||||
->Tag(kKeypointsTestTag)
|
||||
.packets.push_back(PointToForeign(&points).At(Timestamp(1)));
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -589,17 +683,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoMaskDetections) {
|
||||
detections->push_back(detection);
|
||||
|
||||
runner_->MutableInputs()
|
||||
->Tag("CLASS_SEGMENTATION")
|
||||
->Tag(kClassSegmentationTag)
|
||||
.packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -638,17 +732,17 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) {
|
||||
auto flow_ptr =
|
||||
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_flow);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FORWARD_FLOW_ENCODED")
|
||||
->Tag(kForwardFlowEncodedTag)
|
||||
.packets.push_back(Adopt(flow_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -684,11 +778,11 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamNotOK) {
|
||||
auto flow_ptr =
|
||||
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_flow);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FORWARD_FLOW_ENCODED")
|
||||
->Tag(kForwardFlowEncodedTag)
|
||||
.packets.push_back(Adopt(flow_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
absl::Status status = runner_->Run();
|
||||
@@ -705,13 +799,13 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReplacingImages) {
|
||||
mpms::AddImageTimestamp(1, input_sequence.get());
|
||||
mpms::AddImageTimestamp(2, input_sequence.get());
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -731,13 +825,13 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReplacingFlowImages) {
|
||||
mpms::AddForwardFlowTimestamp(1, input_sequence.get());
|
||||
mpms::AddForwardFlowTimestamp(2, input_sequence.get());
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -768,13 +862,52 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReplacingFloatVectors) {
|
||||
mpms::GetFeatureTimestampSize("OTHER", *input_sequence));
|
||||
ASSERT_EQ(num_timesteps,
|
||||
mpms::GetFeatureFloatsSize("OTHER", *input_sequence));
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
|
||||
ASSERT_EQ(0, mpms::GetFeatureTimestampSize("TEST", output_sequence));
|
||||
ASSERT_EQ(0, mpms::GetFeatureFloatsSize("TEST", output_sequence));
|
||||
ASSERT_EQ(0, mpms::GetFeatureTimestampSize("OTHER", output_sequence));
|
||||
ASSERT_EQ(0, mpms::GetFeatureFloatsSize("OTHER", output_sequence));
|
||||
}
|
||||
|
||||
TEST_F(PackMediaSequenceCalculatorTest, TestReplacingBytesVectors) {
|
||||
SetUpCalculator({"BYTES_FEATURE_TEST:test", "BYTES_FEATURE_OTHER:test2"}, {},
|
||||
false, true);
|
||||
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
|
||||
|
||||
int num_timesteps = 2;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto vs_ptr = ::absl::make_unique<std::vector<std::string>>(
|
||||
2, absl::StrCat("foo", 2 << i));
|
||||
mpms::AddFeatureBytes("TEST", *vs_ptr, input_sequence.get());
|
||||
mpms::AddFeatureTimestamp("TEST", i, input_sequence.get());
|
||||
vs_ptr = ::absl::make_unique<std::vector<std::string>>(
|
||||
2, absl::StrCat("bar", 2 << i));
|
||||
mpms::AddFeatureBytes("OTHER", *vs_ptr, input_sequence.get());
|
||||
mpms::AddFeatureTimestamp("OTHER", i, input_sequence.get());
|
||||
}
|
||||
ASSERT_EQ(num_timesteps,
|
||||
mpms::GetFeatureTimestampSize("TEST", *input_sequence));
|
||||
ASSERT_EQ(num_timesteps, mpms::GetFeatureBytesSize("TEST", *input_sequence));
|
||||
ASSERT_EQ(num_timesteps,
|
||||
mpms::GetFeatureTimestampSize("OTHER", *input_sequence));
|
||||
ASSERT_EQ(num_timesteps, mpms::GetFeatureBytesSize("OTHER", *input_sequence));
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -800,7 +933,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
|
||||
for (int i = 0; i < num_images; ++i) {
|
||||
auto image_ptr =
|
||||
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
|
||||
runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
|
||||
runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
|
||||
Adopt(image_ptr.release()).At(Timestamp((i + 1) * 10)));
|
||||
}
|
||||
|
||||
@@ -812,11 +945,11 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
|
||||
mpms::AddBBoxTimestamp("PREFIX", 9, input_sequence.get());
|
||||
mpms::AddBBoxTimestamp("PREFIX", 22, input_sequence.get());
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
|
||||
runner_->Outputs().Tag(kSequenceExampleTag).packets;
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const tf::SequenceExample& output_sequence =
|
||||
output_packets[0].Get<tf::SequenceExample>();
|
||||
@@ -853,7 +986,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
|
||||
for (int i = 0; i < num_images; ++i) {
|
||||
auto image_ptr =
|
||||
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
|
||||
runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
|
||||
runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
|
||||
Adopt(image_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
@@ -867,7 +1000,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
|
||||
Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5)
|
||||
.ConvertToProto(detection.mutable_location_data());
|
||||
detections->push_back(detection);
|
||||
runner_->MutableInputs()->Tag("BBOX").packets.push_back(
|
||||
runner_->MutableInputs()->Tag(kBboxTag).packets.push_back(
|
||||
Adopt(detections.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
@@ -883,7 +1016,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
|
||||
mpms::AddBBoxTrackIndex({-1}, input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
// If the all the previous values aren't cleared, this assert will fail.
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
@@ -899,11 +1032,11 @@ TEST_F(PackMediaSequenceCalculatorTest, TestTooLargeInputFailsSoftly) {
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto vf_ptr = ::absl::make_unique<std::vector<float>>(1000000, i);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FLOAT_FEATURE_TEST")
|
||||
->Tag(kFloatFeatureTestTag)
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
ASSERT_FALSE(runner_->Run().ok());
|
||||
}
|
||||
|
||||
@@ -26,6 +26,8 @@ namespace mediapipe {
|
||||
namespace tf = ::tensorflow;
|
||||
namespace {
|
||||
|
||||
constexpr char kReferenceTag[] = "REFERENCE";
|
||||
|
||||
constexpr char kMatrix[] = "MATRIX";
|
||||
constexpr char kTensor[] = "TENSOR";
|
||||
|
||||
@@ -68,7 +70,8 @@ class TensorToMatrixCalculatorTest : public ::testing::Test {
|
||||
if (include_rate) {
|
||||
header->set_packet_rate(1.0);
|
||||
}
|
||||
runner_->MutableInputs()->Tag("REFERENCE").header = Adopt(header.release());
|
||||
runner_->MutableInputs()->Tag(kReferenceTag).header =
|
||||
Adopt(header.release());
|
||||
}
|
||||
|
||||
std::unique_ptr<CalculatorRunner> runner_;
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Calculator converts from one-dimensional Tensor of DT_STRING to
|
||||
// vector<std::string> OR from (batched) two-dimensional Tensor of DT_STRING to
|
||||
// vector<vector<std::string>.
|
||||
|
||||
#include "mediapipe/calculators/tensorflow/tensor_to_vector_string_calculator_options.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
class TensorToVectorStringCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
TensorToVectorStringCalculatorOptions options_;
|
||||
};
|
||||
REGISTER_CALCULATOR(TensorToVectorStringCalculator);
|
||||
|
||||
absl::Status TensorToVectorStringCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
// Start with only one input packet.
|
||||
RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
|
||||
<< "Only one input stream is supported.";
|
||||
cc->Inputs().Index(0).Set<tf::Tensor>(
|
||||
// Input Tensor
|
||||
);
|
||||
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
|
||||
<< "Only one output stream is supported.";
|
||||
const auto& options = cc->Options<TensorToVectorStringCalculatorOptions>();
|
||||
if (options.tensor_is_2d()) {
|
||||
RET_CHECK(!options.flatten_nd());
|
||||
cc->Outputs().Index(0).Set<std::vector<std::vector<std::string>>>(
|
||||
/* "Output vector<vector<std::string>>." */);
|
||||
} else {
|
||||
cc->Outputs().Index(0).Set<std::vector<std::string>>(
|
||||
// Output vector<std::string>.
|
||||
);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status TensorToVectorStringCalculator::Open(CalculatorContext* cc) {
|
||||
options_ = cc->Options<TensorToVectorStringCalculatorOptions>();
|
||||
|
||||
// Inform mediapipe that this calculator produces an output at time t for
|
||||
// each input received at time t (i.e. this calculator does not buffer
|
||||
// inputs). This enables mediapipe to propagate time of arrival estimates in
|
||||
// mediapipe graphs through this calculator.
|
||||
cc->SetOffset(/*offset=*/0);
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status TensorToVectorStringCalculator::Process(CalculatorContext* cc) {
|
||||
const tf::Tensor& input_tensor =
|
||||
cc->Inputs().Index(0).Value().Get<tf::Tensor>();
|
||||
RET_CHECK(tf::DT_STRING == input_tensor.dtype())
|
||||
<< "expected DT_STRING input but got "
|
||||
<< tensorflow::DataTypeString(input_tensor.dtype());
|
||||
|
||||
if (options_.tensor_is_2d()) {
|
||||
RET_CHECK(2 == input_tensor.dims())
|
||||
<< "Expected 2-dimensional Tensor, but the tensor shape is: "
|
||||
<< input_tensor.shape().DebugString();
|
||||
auto output = absl::make_unique<std::vector<std::vector<std::string>>>(
|
||||
input_tensor.dim_size(0),
|
||||
std::vector<std::string>(input_tensor.dim_size(1)));
|
||||
for (int i = 0; i < input_tensor.dim_size(0); ++i) {
|
||||
auto& instance_output = output->at(i);
|
||||
const auto& slice =
|
||||
input_tensor.Slice(i, i + 1).unaligned_flat<tensorflow::tstring>();
|
||||
for (int j = 0; j < input_tensor.dim_size(1); ++j) {
|
||||
instance_output.at(j) = slice(j);
|
||||
}
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
} else {
|
||||
if (!options_.flatten_nd()) {
|
||||
RET_CHECK(1 == input_tensor.dims())
|
||||
<< "`flatten_nd` is not set. Expected 1-dimensional Tensor, but the "
|
||||
<< "tensor shape is: " << input_tensor.shape().DebugString();
|
||||
}
|
||||
auto output =
|
||||
absl::make_unique<std::vector<std::string>>(input_tensor.NumElements());
|
||||
const auto& tensor_values = input_tensor.flat<tensorflow::tstring>();
|
||||
for (int i = 0; i < input_tensor.NumElements(); ++i) {
|
||||
output->at(i) = tensor_values(i);
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message TensorToVectorStringCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional TensorToVectorStringCalculatorOptions ext = 386534187;
|
||||
}
|
||||
|
||||
// If true, unpack a 2d tensor (matrix) into a vector<vector<string>>. If
|
||||
// false, convert a 1d tensor (vector) into a vector<string>.
|
||||
optional bool tensor_is_2d = 1 [default = false];
|
||||
|
||||
// If true, an N-D tensor will be flattened to a vector<string>. This is
|
||||
// exclusive with tensor_is_2d.
|
||||
optional bool flatten_nd = 2 [default = false];
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright 2018 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensorflow/tensor_to_vector_string_calculator_options.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/types.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
class TensorToVectorStringCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUpRunner(const bool tensor_is_2d, const bool flatten_nd) {
|
||||
CalculatorGraphConfig::Node config;
|
||||
config.set_calculator("TensorToVectorStringCalculator");
|
||||
config.add_input_stream("input_tensor");
|
||||
config.add_output_stream("output_tensor");
|
||||
auto options = config.mutable_options()->MutableExtension(
|
||||
TensorToVectorStringCalculatorOptions::ext);
|
||||
options->set_tensor_is_2d(tensor_is_2d);
|
||||
options->set_flatten_nd(flatten_nd);
|
||||
runner_ = absl::make_unique<CalculatorRunner>(config);
|
||||
}
|
||||
|
||||
std::unique_ptr<CalculatorRunner> runner_;
|
||||
};
|
||||
|
||||
TEST_F(TensorToVectorStringCalculatorTest, ConvertsToVectorFloat) {
|
||||
SetUpRunner(false, false);
|
||||
const tf::TensorShape tensor_shape(std::vector<tf::int64>{5});
|
||||
auto tensor = absl::make_unique<tf::Tensor>(tf::DT_STRING, tensor_shape);
|
||||
auto tensor_vec = tensor->vec<tensorflow::tstring>();
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
tensor_vec(i) = absl::StrCat("foo", i);
|
||||
}
|
||||
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(tensor.release()).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const std::vector<std::string>& output_vector =
|
||||
output_packets[0].Get<std::vector<std::string>>();
|
||||
|
||||
EXPECT_EQ(5, output_vector.size());
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
const std::string expected = absl::StrCat("foo", i);
|
||||
EXPECT_EQ(expected, output_vector[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorToVectorStringCalculatorTest, ConvertsBatchedToVectorVectorFloat) {
|
||||
SetUpRunner(true, false);
|
||||
const tf::TensorShape tensor_shape(std::vector<tf::int64>{1, 5});
|
||||
auto tensor = absl::make_unique<tf::Tensor>(tf::DT_STRING, tensor_shape);
|
||||
auto slice = tensor->Slice(0, 1).flat<tensorflow::tstring>();
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
slice(i) = absl::StrCat("foo", i);
|
||||
}
|
||||
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(tensor.release()).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const std::vector<std::vector<std::string>>& output_vectors =
|
||||
output_packets[0].Get<std::vector<std::vector<std::string>>>();
|
||||
ASSERT_EQ(1, output_vectors.size());
|
||||
const std::vector<std::string>& output_vector = output_vectors[0];
|
||||
EXPECT_EQ(5, output_vector.size());
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
const std::string expected = absl::StrCat("foo", i);
|
||||
EXPECT_EQ(expected, output_vector[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorToVectorStringCalculatorTest, FlattenShouldTakeAllDimensions) {
|
||||
SetUpRunner(false, true);
|
||||
const tf::TensorShape tensor_shape(std::vector<tf::int64>{2, 2, 2});
|
||||
auto tensor = absl::make_unique<tf::Tensor>(tf::DT_STRING, tensor_shape);
|
||||
auto slice = tensor->flat<tensorflow::tstring>();
|
||||
for (int i = 0; i < 2 * 2 * 2; ++i) {
|
||||
slice(i) = absl::StrCat("foo", i);
|
||||
}
|
||||
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(tensor.release()).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const std::vector<std::string>& output_vector =
|
||||
output_packets[0].Get<std::vector<std::string>>();
|
||||
EXPECT_EQ(2 * 2 * 2, output_vector.size());
|
||||
for (int i = 0; i < 2 * 2 * 2; ++i) {
|
||||
const std::string expected = absl::StrCat("foo", i);
|
||||
EXPECT_EQ(expected, output_vector[i]);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -49,6 +49,11 @@ namespace tf = ::tensorflow;
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kRecurrentInitTensorsTag[] = "RECURRENT_INIT_TENSORS";
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
constexpr char kSessionBundleTag[] = "SESSION_BUNDLE";
|
||||
|
||||
// This is a simple implementation of a semaphore using standard C++ libraries.
|
||||
// It is supposed to be used only by TensorflowInferenceCalculator to throttle
|
||||
// the concurrent calls of Tensorflow Session::Run. This is useful when multiple
|
||||
@@ -252,10 +257,10 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
}
|
||||
// A mediapipe::TensorFlowSession with a model loaded and ready for use.
|
||||
// For this calculator it must include a tag_to_tensor_map.
|
||||
cc->InputSidePackets().Tag("SESSION").Set<TensorFlowSession>();
|
||||
if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS")) {
|
||||
cc->InputSidePackets().Tag(kSessionTag).Set<TensorFlowSession>();
|
||||
if (cc->InputSidePackets().HasTag(kRecurrentInitTensorsTag)) {
|
||||
cc->InputSidePackets()
|
||||
.Tag("RECURRENT_INIT_TENSORS")
|
||||
.Tag(kRecurrentInitTensorsTag)
|
||||
.Set<std::unique_ptr<std::map<std::string, tf::Tensor>>>();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
@@ -265,11 +270,11 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
|
||||
std::unique_ptr<InferenceState> inference_state =
|
||||
absl::make_unique<InferenceState>();
|
||||
if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS") &&
|
||||
!cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS").IsEmpty()) {
|
||||
if (cc->InputSidePackets().HasTag(kRecurrentInitTensorsTag) &&
|
||||
!cc->InputSidePackets().Tag(kRecurrentInitTensorsTag).IsEmpty()) {
|
||||
std::map<std::string, tf::Tensor>* init_tensor_map;
|
||||
init_tensor_map = GetFromUniquePtr<std::map<std::string, tf::Tensor>>(
|
||||
cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS"));
|
||||
cc->InputSidePackets().Tag(kRecurrentInitTensorsTag));
|
||||
for (const auto& p : *init_tensor_map) {
|
||||
inference_state->input_tensor_batches_[p.first].emplace_back(p.second);
|
||||
}
|
||||
@@ -280,13 +285,13 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
absl::Status Open(CalculatorContext* cc) override {
|
||||
options_ = cc->Options<TensorFlowInferenceCalculatorOptions>();
|
||||
|
||||
RET_CHECK(cc->InputSidePackets().HasTag("SESSION"));
|
||||
RET_CHECK(cc->InputSidePackets().HasTag(kSessionTag));
|
||||
session_ = cc->InputSidePackets()
|
||||
.Tag("SESSION")
|
||||
.Tag(kSessionTag)
|
||||
.Get<TensorFlowSession>()
|
||||
.session.get();
|
||||
tag_to_tensor_map_ = cc->InputSidePackets()
|
||||
.Tag("SESSION")
|
||||
.Tag(kSessionTag)
|
||||
.Get<TensorFlowSession>()
|
||||
.tag_to_tensor_map;
|
||||
|
||||
|
||||
@@ -41,6 +41,11 @@ namespace mediapipe {
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kMultipliedTag[] = "MULTIPLIED";
|
||||
constexpr char kBTag[] = "B";
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
|
||||
std::string GetGraphDefPath() {
|
||||
#ifdef __APPLE__
|
||||
char path[1024];
|
||||
@@ -86,8 +91,8 @@ class TensorflowInferenceCalculatorTest : public ::testing::Test {
|
||||
MEDIAPIPE_CHECK_OK(tool::RunGenerateAndValidateTypes(
|
||||
"TensorFlowSessionFromFrozenGraphGenerator", extendable_options,
|
||||
input_side_packets, &output_side_packets));
|
||||
runner_->MutableSidePackets()->Tag("SESSION") =
|
||||
output_side_packets.Tag("SESSION");
|
||||
runner_->MutableSidePackets()->Tag(kSessionTag) =
|
||||
output_side_packets.Tag(kSessionTag);
|
||||
}
|
||||
|
||||
Packet CreateTensorPacket(const std::vector<int32>& input, int64 time) {
|
||||
@@ -140,7 +145,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetConstants) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_b =
|
||||
runner_->Outputs().Tag("B").packets;
|
||||
runner_->Outputs().Tag(kBTag).packets;
|
||||
ASSERT_EQ(output_packets_b.size(), 1);
|
||||
const tf::Tensor& tensor_b = output_packets_b[0].Get<tf::Tensor>();
|
||||
tf::TensorShape expected_shape({1, 3});
|
||||
@@ -148,7 +153,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetConstants) {
|
||||
tf::test::ExpectTensorEqual<int32>(expected_tensor, tensor_b);
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(1, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
expected_tensor = tf::test::AsTensor<int32>({0, 0, 0}, expected_shape);
|
||||
@@ -181,7 +186,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetComputed) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(1, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
tf::TensorShape expected_shape({3});
|
||||
@@ -220,7 +225,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetComputed_MaxInFlight) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(1, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
tf::TensorShape expected_shape({3});
|
||||
@@ -274,7 +279,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetMultiBatchComputed) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(2, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
|
||||
@@ -311,7 +316,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetMultiBatchComputed_MaxInFlight) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(2, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
|
||||
@@ -351,7 +356,7 @@ TEST_F(TensorflowInferenceCalculatorTest,
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(3, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
|
||||
@@ -392,7 +397,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetSingleBatchComputed) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(2, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
|
||||
@@ -430,7 +435,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetCloseBatchComputed) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(2, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
|
||||
@@ -481,7 +486,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetBatchComputed_MaxInFlight) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(5, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
|
||||
@@ -528,7 +533,7 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStates) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(2, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
LOG(INFO) << "timestamp: " << 0;
|
||||
@@ -569,7 +574,7 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStateOverride) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(2, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
LOG(INFO) << "timestamp: " << 0;
|
||||
@@ -662,7 +667,7 @@ TEST_F(TensorflowInferenceCalculatorTest, MissingInputFeature_Skip) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(0, output_packets_mult.size());
|
||||
}
|
||||
|
||||
@@ -691,7 +696,7 @@ TEST_F(TensorflowInferenceCalculatorTest,
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets_mult =
|
||||
runner_->Outputs().Tag("MULTIPLIED").packets;
|
||||
runner_->Outputs().Tag(kMultipliedTag).packets;
|
||||
ASSERT_EQ(1, output_packets_mult.size());
|
||||
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
|
||||
auto expected_tensor = tf::test::AsTensor<int32>({9, 12, 15});
|
||||
|
||||
+24
-17
@@ -47,6 +47,11 @@ namespace mediapipe {
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
constexpr char kStringModelFilePathTag[] = "STRING_MODEL_FILE_PATH";
|
||||
constexpr char kStringModelTag[] = "STRING_MODEL";
|
||||
|
||||
// Updates the graph nodes to use the device as specified by device_id.
|
||||
void SetPreferredDevice(tf::GraphDef* graph_def, absl::string_view device_id) {
|
||||
for (auto& node : *graph_def->mutable_node()) {
|
||||
@@ -64,30 +69,32 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
|
||||
cc->Options<TensorFlowSessionFromFrozenGraphCalculatorOptions>();
|
||||
bool has_exactly_one_model =
|
||||
!options.graph_proto_path().empty()
|
||||
? !(cc->InputSidePackets().HasTag("STRING_MODEL") |
|
||||
cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"))
|
||||
: (cc->InputSidePackets().HasTag("STRING_MODEL") ^
|
||||
cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"));
|
||||
? !(cc->InputSidePackets().HasTag(kStringModelTag) |
|
||||
cc->InputSidePackets().HasTag(kStringModelFilePathTag))
|
||||
: (cc->InputSidePackets().HasTag(kStringModelTag) ^
|
||||
cc->InputSidePackets().HasTag(kStringModelFilePathTag));
|
||||
RET_CHECK(has_exactly_one_model)
|
||||
<< "Must have exactly one of graph_proto_path in options or "
|
||||
"input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH";
|
||||
if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
|
||||
if (cc->InputSidePackets().HasTag(kStringModelTag)) {
|
||||
cc->InputSidePackets()
|
||||
.Tag("STRING_MODEL")
|
||||
.Tag(kStringModelTag)
|
||||
.Set<std::string>(
|
||||
// String model from embedded path
|
||||
);
|
||||
} else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
|
||||
} else if (cc->InputSidePackets().HasTag(kStringModelFilePathTag)) {
|
||||
cc->InputSidePackets()
|
||||
.Tag("STRING_MODEL_FILE_PATH")
|
||||
.Tag(kStringModelFilePathTag)
|
||||
.Set<std::string>(
|
||||
// Filename of std::string model.
|
||||
);
|
||||
}
|
||||
cc->OutputSidePackets().Tag("SESSION").Set<TensorFlowSession>(
|
||||
// A TensorFlow model loaded and ready for use along with
|
||||
// a map from tags to tensor names.
|
||||
);
|
||||
cc->OutputSidePackets()
|
||||
.Tag(kSessionTag)
|
||||
.Set<TensorFlowSession>(
|
||||
// A TensorFlow model loaded and ready for use along with
|
||||
// a map from tags to tensor names.
|
||||
);
|
||||
RET_CHECK_GT(options.tag_to_tensor_names().size(), 0);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
@@ -111,12 +118,12 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
|
||||
session->session.reset(tf::NewSession(session_options));
|
||||
|
||||
std::string graph_def_serialized;
|
||||
if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
|
||||
if (cc->InputSidePackets().HasTag(kStringModelTag)) {
|
||||
graph_def_serialized =
|
||||
cc->InputSidePackets().Tag("STRING_MODEL").Get<std::string>();
|
||||
} else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
|
||||
cc->InputSidePackets().Tag(kStringModelTag).Get<std::string>();
|
||||
} else if (cc->InputSidePackets().HasTag(kStringModelFilePathTag)) {
|
||||
const std::string& frozen_graph = cc->InputSidePackets()
|
||||
.Tag("STRING_MODEL_FILE_PATH")
|
||||
.Tag(kStringModelFilePathTag)
|
||||
.Get<std::string>();
|
||||
RET_CHECK_OK(
|
||||
mediapipe::file::GetContents(frozen_graph, &graph_def_serialized));
|
||||
@@ -147,7 +154,7 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
|
||||
RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
|
||||
}
|
||||
|
||||
cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release()));
|
||||
cc->OutputSidePackets().Tag(kSessionTag).Set(Adopt(session.release()));
|
||||
const uint64 end_time = absl::ToUnixMicros(clock->TimeNow());
|
||||
LOG(INFO) << "Loaded frozen model in: " << end_time - start_time
|
||||
<< " microseconds.";
|
||||
|
||||
+15
-11
@@ -37,6 +37,10 @@ namespace {
|
||||
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
constexpr char kStringModelFilePathTag[] = "STRING_MODEL_FILE_PATH";
|
||||
constexpr char kStringModelTag[] = "STRING_MODEL";
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
|
||||
std::string GetGraphDefPath() {
|
||||
return mediapipe::file::JoinPath("./",
|
||||
"mediapipe/calculators/tensorflow/"
|
||||
@@ -112,7 +116,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const TensorFlowSession& session =
|
||||
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
|
||||
runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
VerifySignatureMap(session);
|
||||
}
|
||||
|
||||
@@ -190,12 +194,12 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
|
||||
std::string serialized_graph_contents;
|
||||
MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
|
||||
&serialized_graph_contents));
|
||||
runner.MutableSidePackets()->Tag("STRING_MODEL") =
|
||||
runner.MutableSidePackets()->Tag(kStringModelTag) =
|
||||
Adopt(new std::string(serialized_graph_contents));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const TensorFlowSession& session =
|
||||
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
|
||||
runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
VerifySignatureMap(session);
|
||||
}
|
||||
|
||||
@@ -213,12 +217,12 @@ TEST_F(
|
||||
}
|
||||
})",
|
||||
calculator_options_->DebugString()));
|
||||
runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
|
||||
runner.MutableSidePackets()->Tag(kStringModelFilePathTag) =
|
||||
Adopt(new std::string(GetGraphDefPath()));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const TensorFlowSession& session =
|
||||
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
|
||||
runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
VerifySignatureMap(session);
|
||||
}
|
||||
|
||||
@@ -234,7 +238,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
|
||||
}
|
||||
})",
|
||||
calculator_options_->DebugString()));
|
||||
runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
|
||||
runner.MutableSidePackets()->Tag(kStringModelFilePathTag) =
|
||||
Adopt(new std::string(GetGraphDefPath()));
|
||||
auto run_status = runner.Run();
|
||||
EXPECT_THAT(
|
||||
@@ -255,12 +259,12 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
|
||||
}
|
||||
})",
|
||||
calculator_options_->DebugString()));
|
||||
runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
|
||||
runner.MutableSidePackets()->Tag(kStringModelFilePathTag) =
|
||||
Adopt(new std::string(GetGraphDefPath()));
|
||||
std::string serialized_graph_contents;
|
||||
MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
|
||||
&serialized_graph_contents));
|
||||
runner.MutableSidePackets()->Tag("STRING_MODEL") =
|
||||
runner.MutableSidePackets()->Tag(kStringModelTag) =
|
||||
Adopt(new std::string(serialized_graph_contents));
|
||||
auto run_status = runner.Run();
|
||||
EXPECT_THAT(
|
||||
@@ -282,12 +286,12 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
|
||||
}
|
||||
})",
|
||||
calculator_options_->DebugString()));
|
||||
runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
|
||||
runner.MutableSidePackets()->Tag(kStringModelFilePathTag) =
|
||||
Adopt(new std::string(GetGraphDefPath()));
|
||||
std::string serialized_graph_contents;
|
||||
MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
|
||||
&serialized_graph_contents));
|
||||
runner.MutableSidePackets()->Tag("STRING_MODEL") =
|
||||
runner.MutableSidePackets()->Tag(kStringModelTag) =
|
||||
Adopt(new std::string(serialized_graph_contents));
|
||||
auto run_status = runner.Run();
|
||||
EXPECT_THAT(
|
||||
@@ -310,7 +314,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const TensorFlowSession& session =
|
||||
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
|
||||
runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
VerifySignatureMap(session);
|
||||
}
|
||||
|
||||
|
||||
+23
-17
@@ -43,6 +43,11 @@ namespace mediapipe {
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
constexpr char kStringModelFilePathTag[] = "STRING_MODEL_FILE_PATH";
|
||||
constexpr char kStringModelTag[] = "STRING_MODEL";
|
||||
|
||||
// Updates the graph nodes to use the device as specified by device_id.
|
||||
void SetPreferredDevice(tf::GraphDef* graph_def, absl::string_view device_id) {
|
||||
for (auto& node : *graph_def->mutable_node()) {
|
||||
@@ -64,28 +69,29 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
|
||||
TensorFlowSessionFromFrozenGraphGeneratorOptions::ext);
|
||||
bool has_exactly_one_model =
|
||||
!options.graph_proto_path().empty()
|
||||
? !(input_side_packets->HasTag("STRING_MODEL") |
|
||||
input_side_packets->HasTag("STRING_MODEL_FILE_PATH"))
|
||||
: (input_side_packets->HasTag("STRING_MODEL") ^
|
||||
input_side_packets->HasTag("STRING_MODEL_FILE_PATH"));
|
||||
? !(input_side_packets->HasTag(kStringModelTag) |
|
||||
input_side_packets->HasTag(kStringModelFilePathTag))
|
||||
: (input_side_packets->HasTag(kStringModelTag) ^
|
||||
input_side_packets->HasTag(kStringModelFilePathTag));
|
||||
RET_CHECK(has_exactly_one_model)
|
||||
<< "Must have exactly one of graph_proto_path in options or "
|
||||
"input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH";
|
||||
if (input_side_packets->HasTag("STRING_MODEL")) {
|
||||
input_side_packets->Tag("STRING_MODEL")
|
||||
if (input_side_packets->HasTag(kStringModelTag)) {
|
||||
input_side_packets->Tag(kStringModelTag)
|
||||
.Set<std::string>(
|
||||
// String model from embedded path
|
||||
);
|
||||
} else if (input_side_packets->HasTag("STRING_MODEL_FILE_PATH")) {
|
||||
input_side_packets->Tag("STRING_MODEL_FILE_PATH")
|
||||
} else if (input_side_packets->HasTag(kStringModelFilePathTag)) {
|
||||
input_side_packets->Tag(kStringModelFilePathTag)
|
||||
.Set<std::string>(
|
||||
// Filename of std::string model.
|
||||
);
|
||||
}
|
||||
output_side_packets->Tag("SESSION").Set<TensorFlowSession>(
|
||||
// A TensorFlow model loaded and ready for use along with
|
||||
// a map from tags to tensor names.
|
||||
);
|
||||
output_side_packets->Tag(kSessionTag)
|
||||
.Set<TensorFlowSession>(
|
||||
// A TensorFlow model loaded and ready for use along with
|
||||
// a map from tags to tensor names.
|
||||
);
|
||||
RET_CHECK_GT(options.tag_to_tensor_names().size(), 0);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
@@ -112,12 +118,12 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
|
||||
session->session.reset(tf::NewSession(session_options));
|
||||
|
||||
std::string graph_def_serialized;
|
||||
if (input_side_packets.HasTag("STRING_MODEL")) {
|
||||
if (input_side_packets.HasTag(kStringModelTag)) {
|
||||
graph_def_serialized =
|
||||
input_side_packets.Tag("STRING_MODEL").Get<std::string>();
|
||||
} else if (input_side_packets.HasTag("STRING_MODEL_FILE_PATH")) {
|
||||
input_side_packets.Tag(kStringModelTag).Get<std::string>();
|
||||
} else if (input_side_packets.HasTag(kStringModelFilePathTag)) {
|
||||
const std::string& frozen_graph =
|
||||
input_side_packets.Tag("STRING_MODEL_FILE_PATH").Get<std::string>();
|
||||
input_side_packets.Tag(kStringModelFilePathTag).Get<std::string>();
|
||||
RET_CHECK_OK(
|
||||
mediapipe::file::GetContents(frozen_graph, &graph_def_serialized));
|
||||
} else {
|
||||
@@ -147,7 +153,7 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
|
||||
RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
|
||||
}
|
||||
|
||||
output_side_packets->Tag("SESSION") = Adopt(session.release());
|
||||
output_side_packets->Tag(kSessionTag) = Adopt(session.release());
|
||||
const uint64 end_time = absl::ToUnixMicros(clock->TimeNow());
|
||||
LOG(INFO) << "Loaded frozen model in: " << end_time - start_time
|
||||
<< " microseconds.";
|
||||
|
||||
+12
-8
@@ -37,6 +37,10 @@ namespace {
|
||||
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
constexpr char kStringModelFilePathTag[] = "STRING_MODEL_FILE_PATH";
|
||||
constexpr char kStringModelTag[] = "STRING_MODEL";
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
|
||||
std::string GetGraphDefPath() {
|
||||
return mediapipe::file::JoinPath("./",
|
||||
"mediapipe/calculators/tensorflow/"
|
||||
@@ -72,7 +76,7 @@ class TensorFlowSessionFromFrozenGraphGeneratorTest : public ::testing::Test {
|
||||
|
||||
void VerifySignatureMap(PacketSet* output_side_packets) {
|
||||
const TensorFlowSession& session =
|
||||
output_side_packets->Tag("SESSION").Get<TensorFlowSession>();
|
||||
output_side_packets->Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
// Session must be set.
|
||||
ASSERT_NE(session.session, nullptr);
|
||||
|
||||
@@ -179,7 +183,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
|
||||
MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
|
||||
&serialized_graph_contents));
|
||||
generator_options_->clear_graph_proto_path();
|
||||
input_side_packets.Tag("STRING_MODEL") =
|
||||
input_side_packets.Tag(kStringModelTag) =
|
||||
Adopt(new std::string(serialized_graph_contents));
|
||||
absl::Status run_status = tool::RunGenerateAndValidateTypes(
|
||||
"TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
|
||||
@@ -196,7 +200,7 @@ TEST_F(
|
||||
PacketSet output_side_packets(
|
||||
tool::CreateTagMap({"SESSION:session"}).value());
|
||||
generator_options_->clear_graph_proto_path();
|
||||
input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
|
||||
input_side_packets.Tag(kStringModelFilePathTag) =
|
||||
Adopt(new std::string(GetGraphDefPath()));
|
||||
absl::Status run_status = tool::RunGenerateAndValidateTypes(
|
||||
"TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
|
||||
@@ -211,7 +215,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
|
||||
tool::CreateTagMap({"STRING_MODEL_FILE_PATH:model_path"}).value());
|
||||
PacketSet output_side_packets(
|
||||
tool::CreateTagMap({"SESSION:session"}).value());
|
||||
input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
|
||||
input_side_packets.Tag(kStringModelFilePathTag) =
|
||||
Adopt(new std::string(GetGraphDefPath()));
|
||||
absl::Status run_status = tool::RunGenerateAndValidateTypes(
|
||||
"TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
|
||||
@@ -233,9 +237,9 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
|
||||
std::string serialized_graph_contents;
|
||||
MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
|
||||
&serialized_graph_contents));
|
||||
input_side_packets.Tag("STRING_MODEL") =
|
||||
input_side_packets.Tag(kStringModelTag) =
|
||||
Adopt(new std::string(serialized_graph_contents));
|
||||
input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
|
||||
input_side_packets.Tag(kStringModelFilePathTag) =
|
||||
Adopt(new std::string(GetGraphDefPath()));
|
||||
|
||||
absl::Status run_status = tool::RunGenerateAndValidateTypes(
|
||||
@@ -258,9 +262,9 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
|
||||
std::string serialized_graph_contents;
|
||||
MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
|
||||
&serialized_graph_contents));
|
||||
input_side_packets.Tag("STRING_MODEL") =
|
||||
input_side_packets.Tag(kStringModelTag) =
|
||||
Adopt(new std::string(serialized_graph_contents));
|
||||
input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
|
||||
input_side_packets.Tag(kStringModelFilePathTag) =
|
||||
Adopt(new std::string(GetGraphDefPath()));
|
||||
generator_options_->clear_graph_proto_path();
|
||||
|
||||
|
||||
@@ -31,6 +31,9 @@
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
|
||||
static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
|
||||
|
||||
// Given the path to a directory containing multiple tensorflow saved models
|
||||
@@ -108,7 +111,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase {
|
||||
cc->InputSidePackets().Tag(kStringSavedModelPath).Set<std::string>();
|
||||
}
|
||||
// A TensorFlow model loaded and ready for use along with tensor
|
||||
cc->OutputSidePackets().Tag("SESSION").Set<TensorFlowSession>();
|
||||
cc->OutputSidePackets().Tag(kSessionTag).Set<TensorFlowSession>();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -160,7 +163,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase {
|
||||
output_signature.first, options)] = output_signature.second.name();
|
||||
}
|
||||
|
||||
cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release()));
|
||||
cc->OutputSidePackets().Tag(kSessionTag).Set(Adopt(session.release()));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
|
||||
+8
-5
@@ -35,6 +35,9 @@ namespace {
|
||||
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
constexpr char kStringSavedModelPathTag[] = "STRING_SAVED_MODEL_PATH";
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
|
||||
std::string GetSavedModelDir() {
|
||||
std::string out_path =
|
||||
file::JoinPath("./", "mediapipe/calculators/tensorflow/testdata/",
|
||||
@@ -79,7 +82,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
|
||||
options_->DebugString()));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const TensorFlowSession& session =
|
||||
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
|
||||
runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
// Session must be set.
|
||||
ASSERT_NE(session.session, nullptr);
|
||||
|
||||
@@ -119,11 +122,11 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
|
||||
}
|
||||
})",
|
||||
options_->DebugString()));
|
||||
runner.MutableSidePackets()->Tag("STRING_SAVED_MODEL_PATH") =
|
||||
runner.MutableSidePackets()->Tag(kStringSavedModelPathTag) =
|
||||
MakePacket<std::string>(GetSavedModelDir());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const TensorFlowSession& session =
|
||||
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
|
||||
runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
// Session must be set.
|
||||
ASSERT_NE(session.session, nullptr);
|
||||
}
|
||||
@@ -201,7 +204,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
|
||||
options_->DebugString()));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const TensorFlowSession& session =
|
||||
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
|
||||
runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
// Session must be set.
|
||||
ASSERT_NE(session.session, nullptr);
|
||||
}
|
||||
@@ -224,7 +227,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
|
||||
options_->DebugString()));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const TensorFlowSession& session =
|
||||
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
|
||||
runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
// Session must be set.
|
||||
ASSERT_NE(session.session, nullptr);
|
||||
std::vector<tensorflow::DeviceAttributes> devices;
|
||||
|
||||
@@ -33,6 +33,9 @@
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
|
||||
static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
|
||||
|
||||
// Given the path to a directory containing multiple tensorflow saved models
|
||||
@@ -100,7 +103,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator {
|
||||
input_side_packets->Tag(kStringSavedModelPath).Set<std::string>();
|
||||
}
|
||||
// A TensorFlow model loaded and ready for use along with tensor
|
||||
output_side_packets->Tag("SESSION").Set<TensorFlowSession>();
|
||||
output_side_packets->Tag(kSessionTag).Set<TensorFlowSession>();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -153,7 +156,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator {
|
||||
output_signature.first, options)] = output_signature.second.name();
|
||||
}
|
||||
|
||||
output_side_packets->Tag("SESSION") = Adopt(session.release());
|
||||
output_side_packets->Tag(kSessionTag) = Adopt(session.release());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
+8
-5
@@ -34,6 +34,9 @@ namespace {
|
||||
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
constexpr char kStringSavedModelPathTag[] = "STRING_SAVED_MODEL_PATH";
|
||||
constexpr char kSessionTag[] = "SESSION";
|
||||
|
||||
std::string GetSavedModelDir() {
|
||||
std::string out_path =
|
||||
file::JoinPath("./", "mediapipe/calculators/tensorflow/testdata/",
|
||||
@@ -75,7 +78,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
|
||||
input_side_packets, &output_side_packets);
|
||||
MP_EXPECT_OK(run_status) << run_status.message();
|
||||
const TensorFlowSession& session =
|
||||
output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
|
||||
output_side_packets.Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
// Session must be set.
|
||||
ASSERT_NE(session.session, nullptr);
|
||||
|
||||
@@ -107,7 +110,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
|
||||
generator_options_->clear_saved_model_path();
|
||||
PacketSet input_side_packets(
|
||||
tool::CreateTagMap({"STRING_SAVED_MODEL_PATH:saved_model_dir"}).value());
|
||||
input_side_packets.Tag("STRING_SAVED_MODEL_PATH") =
|
||||
input_side_packets.Tag(kStringSavedModelPathTag) =
|
||||
Adopt(new std::string(GetSavedModelDir()));
|
||||
PacketSet output_side_packets(
|
||||
tool::CreateTagMap({"SESSION:session"}).value());
|
||||
@@ -116,7 +119,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
|
||||
input_side_packets, &output_side_packets);
|
||||
MP_EXPECT_OK(run_status) << run_status.message();
|
||||
const TensorFlowSession& session =
|
||||
output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
|
||||
output_side_packets.Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
// Session must be set.
|
||||
ASSERT_NE(session.session, nullptr);
|
||||
}
|
||||
@@ -192,7 +195,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
|
||||
input_side_packets, &output_side_packets);
|
||||
MP_EXPECT_OK(run_status) << run_status.message();
|
||||
const TensorFlowSession& session =
|
||||
output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
|
||||
output_side_packets.Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
// Session must be set.
|
||||
ASSERT_NE(session.session, nullptr);
|
||||
}
|
||||
@@ -213,7 +216,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
|
||||
input_side_packets, &output_side_packets);
|
||||
MP_EXPECT_OK(run_status) << run_status.message();
|
||||
const TensorFlowSession& session =
|
||||
output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
|
||||
output_side_packets.Tag(kSessionTag).Get<TensorFlowSession>();
|
||||
// Session must be set.
|
||||
ASSERT_NE(session.session, nullptr);
|
||||
std::vector<tensorflow::DeviceAttributes> devices;
|
||||
|
||||
@@ -33,6 +33,31 @@ namespace {
|
||||
namespace tf = ::tensorflow;
|
||||
namespace mpms = mediapipe::mediasequence;
|
||||
|
||||
constexpr char kImageFrameRateTag[] = "IMAGE_FRAME_RATE";
|
||||
constexpr char kEncodedMediaStartTimestampTag[] =
|
||||
"ENCODED_MEDIA_START_TIMESTAMP";
|
||||
constexpr char kEncodedMediaTag[] = "ENCODED_MEDIA";
|
||||
constexpr char kResamplerOptionsTag[] = "RESAMPLER_OPTIONS";
|
||||
constexpr char kSandboxedDecoderOptionsTag[] = "SANDBOXED_DECODER_OPTIONS";
|
||||
constexpr char kDecoderOptionsTag[] = "DECODER_OPTIONS";
|
||||
constexpr char kAudioDecoderOptionsTag[] = "AUDIO_DECODER_OPTIONS";
|
||||
constexpr char kDataPathTag[] = "DATA_PATH";
|
||||
constexpr char kDatasetRootTag[] = "DATASET_ROOT";
|
||||
constexpr char kMediaIdTag[] = "MEDIA_ID";
|
||||
constexpr char kFloatFeatureFdenseMaxTag[] = "FLOAT_FEATURE_FDENSE_MAX";
|
||||
constexpr char kFloatFeatureFdenseAvgTag[] = "FLOAT_FEATURE_FDENSE_AVG";
|
||||
constexpr char kAudioOtherTag[] = "AUDIO_OTHER";
|
||||
constexpr char kAudioTestTag[] = "AUDIO_TEST";
|
||||
constexpr char kFloatFeatureOtherTag[] = "FLOAT_FEATURE_OTHER";
|
||||
constexpr char kFloatFeatureTestTag[] = "FLOAT_FEATURE_TEST";
|
||||
constexpr char kBboxPrefixTag[] = "BBOX_PREFIX";
|
||||
constexpr char kKeypointsTag[] = "KEYPOINTS";
|
||||
constexpr char kBboxTag[] = "BBOX";
|
||||
constexpr char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
|
||||
constexpr char kImagePrefixTag[] = "IMAGE_PREFIX";
|
||||
constexpr char kImageTag[] = "IMAGE";
|
||||
constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
|
||||
|
||||
class UnpackMediaSequenceCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUpCalculator(const std::vector<std::string>& output_streams,
|
||||
@@ -95,13 +120,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksOneImage) {
|
||||
mpms::AddImageEncoded(test_image_string, input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("IMAGE").packets;
|
||||
runner_->Outputs().Tag(kImageTag).packets;
|
||||
ASSERT_EQ(num_images, output_packets.size());
|
||||
|
||||
for (int i = 0; i < num_images; ++i) {
|
||||
@@ -124,13 +149,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoImages) {
|
||||
mpms::AddImageEncoded(test_image_string, input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("IMAGE").packets;
|
||||
runner_->Outputs().Tag(kImageTag).packets;
|
||||
ASSERT_EQ(num_images, output_packets.size());
|
||||
|
||||
for (int i = 0; i < num_images; ++i) {
|
||||
@@ -154,13 +179,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPrefixedImages) {
|
||||
mpms::AddImageEncoded(prefix, test_image_string, input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("IMAGE_PREFIX").packets;
|
||||
runner_->Outputs().Tag(kImagePrefixTag).packets;
|
||||
ASSERT_EQ(num_images, output_packets.size());
|
||||
|
||||
for (int i = 0; i < num_images; ++i) {
|
||||
@@ -182,12 +207,12 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksOneForwardFlowImage) {
|
||||
mpms::AddForwardFlowEncoded(test_image_string, input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("FORWARD_FLOW_ENCODED").packets;
|
||||
runner_->Outputs().Tag(kForwardFlowEncodedTag).packets;
|
||||
ASSERT_EQ(num_forward_flow_images, output_packets.size());
|
||||
|
||||
for (int i = 0; i < num_forward_flow_images; ++i) {
|
||||
@@ -211,12 +236,12 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoForwardFlowImages) {
|
||||
mpms::AddForwardFlowEncoded(test_image_strings[i], input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("FORWARD_FLOW_ENCODED").packets;
|
||||
runner_->Outputs().Tag(kForwardFlowEncodedTag).packets;
|
||||
ASSERT_EQ(num_forward_flow_images, output_packets.size());
|
||||
|
||||
for (int i = 0; i < num_forward_flow_images; ++i) {
|
||||
@@ -240,13 +265,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksBBoxes) {
|
||||
mpms::AddBBoxTimestamp(i, input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("BBOX").packets;
|
||||
runner_->Outputs().Tag(kBboxTag).packets;
|
||||
ASSERT_EQ(bboxes.size(), output_packets.size());
|
||||
|
||||
for (int i = 0; i < bboxes.size(); ++i) {
|
||||
@@ -274,13 +299,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPrefixedBBoxes) {
|
||||
mpms::AddBBoxTimestamp(prefix, i, input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("BBOX_PREFIX").packets;
|
||||
runner_->Outputs().Tag(kBboxPrefixTag).packets;
|
||||
ASSERT_EQ(bboxes.size(), output_packets.size());
|
||||
|
||||
for (int i = 0; i < bboxes.size(); ++i) {
|
||||
@@ -306,13 +331,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoFloatLists) {
|
||||
mpms::AddFeatureTimestamp("OTHER", i, input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("FLOAT_FEATURE_TEST").packets;
|
||||
runner_->Outputs().Tag(kFloatFeatureTestTag).packets;
|
||||
ASSERT_EQ(num_float_lists, output_packets.size());
|
||||
|
||||
for (int i = 0; i < num_float_lists; ++i) {
|
||||
@@ -322,7 +347,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoFloatLists) {
|
||||
}
|
||||
|
||||
const std::vector<Packet>& output_packets_other =
|
||||
runner_->Outputs().Tag("FLOAT_FEATURE_OTHER").packets;
|
||||
runner_->Outputs().Tag(kFloatFeatureOtherTag).packets;
|
||||
ASSERT_EQ(num_float_lists, output_packets_other.size());
|
||||
|
||||
for (int i = 0; i < num_float_lists; ++i) {
|
||||
@@ -352,12 +377,12 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksNonOverlappingTimestamps) {
|
||||
mpms::AddFeatureTimestamp("OTHER", i + 5, input_sequence.get());
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("IMAGE").packets;
|
||||
runner_->Outputs().Tag(kImageTag).packets;
|
||||
ASSERT_EQ(num_images, output_packets.size());
|
||||
|
||||
for (int i = 0; i < num_images; ++i) {
|
||||
@@ -366,7 +391,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksNonOverlappingTimestamps) {
|
||||
}
|
||||
|
||||
const std::vector<Packet>& output_packets_other =
|
||||
runner_->Outputs().Tag("FLOAT_FEATURE_OTHER").packets;
|
||||
runner_->Outputs().Tag(kFloatFeatureOtherTag).packets;
|
||||
ASSERT_EQ(num_float_lists, output_packets_other.size());
|
||||
|
||||
for (int i = 0; i < num_float_lists; ++i) {
|
||||
@@ -389,12 +414,12 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPostStreamFloatLists) {
|
||||
mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(),
|
||||
input_sequence.get());
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& fdense_avg_packets =
|
||||
runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_AVG").packets;
|
||||
runner_->Outputs().Tag(kFloatFeatureFdenseAvgTag).packets;
|
||||
ASSERT_EQ(fdense_avg_packets.size(), 1);
|
||||
const auto& fdense_avg_vector =
|
||||
fdense_avg_packets[0].Get<std::vector<float>>();
|
||||
@@ -403,7 +428,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPostStreamFloatLists) {
|
||||
::testing::Eq(Timestamp::PostStream()));
|
||||
|
||||
const std::vector<Packet>& fdense_max_packets =
|
||||
runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_MAX").packets;
|
||||
runner_->Outputs().Tag(kFloatFeatureFdenseMaxTag).packets;
|
||||
ASSERT_EQ(fdense_max_packets.size(), 1);
|
||||
const auto& fdense_max_vector =
|
||||
fdense_max_packets[0].Get<std::vector<float>>();
|
||||
@@ -430,13 +455,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksImageWithPostStreamFloatList) {
|
||||
mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(),
|
||||
input_sequence.get());
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("IMAGE").packets;
|
||||
runner_->Outputs().Tag(kImageTag).packets;
|
||||
ASSERT_EQ(num_images, output_packets.size());
|
||||
|
||||
for (int i = 0; i < num_images; ++i) {
|
||||
@@ -463,13 +488,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPostStreamFloatListWithImage) {
|
||||
mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(),
|
||||
input_sequence.get());
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(input_sequence.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
const std::vector<Packet>& fdense_max_packets =
|
||||
runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_MAX").packets;
|
||||
runner_->Outputs().Tag(kFloatFeatureFdenseMaxTag).packets;
|
||||
ASSERT_EQ(fdense_max_packets.size(), 1);
|
||||
const auto& fdense_max_vector =
|
||||
fdense_max_packets[0].Get<std::vector<float>>();
|
||||
@@ -481,17 +506,17 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPostStreamFloatListWithImage) {
|
||||
TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromPacket) {
|
||||
SetUpCalculator({}, {"DATA_PATH:data_path"}, {"DATASET_ROOT:root"});
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(sequence_.release());
|
||||
|
||||
std::string root = "test_root";
|
||||
runner_->MutableSidePackets()->Tag("DATASET_ROOT") = PointToForeign(&root);
|
||||
runner_->MutableSidePackets()->Tag(kDatasetRootTag) = PointToForeign(&root);
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
MP_ASSERT_OK(runner_->OutputSidePackets()
|
||||
.Tag("DATA_PATH")
|
||||
.Tag(kDataPathTag)
|
||||
.ValidateAsType<std::string>());
|
||||
ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get<std::string>(),
|
||||
ASSERT_EQ(runner_->OutputSidePackets().Tag(kDataPathTag).Get<std::string>(),
|
||||
root + "/" + data_path_);
|
||||
}
|
||||
|
||||
@@ -501,28 +526,28 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromOptions) {
|
||||
options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext)
|
||||
->set_dataset_root_directory(root);
|
||||
SetUpCalculator({}, {"DATA_PATH:data_path"}, {}, &options);
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(sequence_.release());
|
||||
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
MP_ASSERT_OK(runner_->OutputSidePackets()
|
||||
.Tag("DATA_PATH")
|
||||
.Tag(kDataPathTag)
|
||||
.ValidateAsType<std::string>());
|
||||
ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get<std::string>(),
|
||||
ASSERT_EQ(runner_->OutputSidePackets().Tag(kDataPathTag).Get<std::string>(),
|
||||
root + "/" + data_path_);
|
||||
}
|
||||
|
||||
TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromExample) {
|
||||
SetUpCalculator({}, {"DATA_PATH:data_path"});
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(sequence_.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
MP_ASSERT_OK(runner_->OutputSidePackets()
|
||||
.Tag("DATA_PATH")
|
||||
.Tag(kDataPathTag)
|
||||
.ValidateAsType<std::string>());
|
||||
ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get<std::string>(),
|
||||
ASSERT_EQ(runner_->OutputSidePackets().Tag(kDataPathTag).Get<std::string>(),
|
||||
data_path_);
|
||||
}
|
||||
|
||||
@@ -534,20 +559,20 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetAudioDecoderOptions) {
|
||||
->set_padding_after_label(2);
|
||||
SetUpCalculator({}, {"AUDIO_DECODER_OPTIONS:audio_decoder_options"}, {},
|
||||
&options);
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(sequence_.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
MP_EXPECT_OK(runner_->OutputSidePackets()
|
||||
.Tag("AUDIO_DECODER_OPTIONS")
|
||||
.Tag(kAudioDecoderOptionsTag)
|
||||
.ValidateAsType<AudioDecoderOptions>());
|
||||
EXPECT_NEAR(runner_->OutputSidePackets()
|
||||
.Tag("AUDIO_DECODER_OPTIONS")
|
||||
.Tag(kAudioDecoderOptionsTag)
|
||||
.Get<AudioDecoderOptions>()
|
||||
.start_time(),
|
||||
2.0, 1e-5);
|
||||
EXPECT_NEAR(runner_->OutputSidePackets()
|
||||
.Tag("AUDIO_DECODER_OPTIONS")
|
||||
.Tag(kAudioDecoderOptionsTag)
|
||||
.Get<AudioDecoderOptions>()
|
||||
.end_time(),
|
||||
7.0, 1e-5);
|
||||
@@ -563,20 +588,20 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetAudioDecoderOptionsOverride) {
|
||||
->set_force_decoding_from_start_of_media(true);
|
||||
SetUpCalculator({}, {"AUDIO_DECODER_OPTIONS:audio_decoder_options"}, {},
|
||||
&options);
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(sequence_.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
MP_EXPECT_OK(runner_->OutputSidePackets()
|
||||
.Tag("AUDIO_DECODER_OPTIONS")
|
||||
.Tag(kAudioDecoderOptionsTag)
|
||||
.ValidateAsType<AudioDecoderOptions>());
|
||||
EXPECT_NEAR(runner_->OutputSidePackets()
|
||||
.Tag("AUDIO_DECODER_OPTIONS")
|
||||
.Tag(kAudioDecoderOptionsTag)
|
||||
.Get<AudioDecoderOptions>()
|
||||
.start_time(),
|
||||
0.0, 1e-5);
|
||||
EXPECT_NEAR(runner_->OutputSidePackets()
|
||||
.Tag("AUDIO_DECODER_OPTIONS")
|
||||
.Tag(kAudioDecoderOptionsTag)
|
||||
.Get<AudioDecoderOptions>()
|
||||
.end_time(),
|
||||
7.0, 1e-5);
|
||||
@@ -594,27 +619,27 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetPacketResamplingOptions) {
|
||||
->mutable_base_packet_resampler_options()
|
||||
->set_frame_rate(1.0);
|
||||
SetUpCalculator({}, {"RESAMPLER_OPTIONS:resampler_options"}, {}, &options);
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(sequence_.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
|
||||
MP_EXPECT_OK(runner_->OutputSidePackets()
|
||||
.Tag("RESAMPLER_OPTIONS")
|
||||
.Tag(kResamplerOptionsTag)
|
||||
.ValidateAsType<CalculatorOptions>());
|
||||
EXPECT_NEAR(runner_->OutputSidePackets()
|
||||
.Tag("RESAMPLER_OPTIONS")
|
||||
.Tag(kResamplerOptionsTag)
|
||||
.Get<CalculatorOptions>()
|
||||
.GetExtension(PacketResamplerCalculatorOptions::ext)
|
||||
.start_time(),
|
||||
2000000, 1);
|
||||
EXPECT_NEAR(runner_->OutputSidePackets()
|
||||
.Tag("RESAMPLER_OPTIONS")
|
||||
.Tag(kResamplerOptionsTag)
|
||||
.Get<CalculatorOptions>()
|
||||
.GetExtension(PacketResamplerCalculatorOptions::ext)
|
||||
.end_time(),
|
||||
7000000, 1);
|
||||
EXPECT_NEAR(runner_->OutputSidePackets()
|
||||
.Tag("RESAMPLER_OPTIONS")
|
||||
.Tag(kResamplerOptionsTag)
|
||||
.Get<CalculatorOptions>()
|
||||
.GetExtension(PacketResamplerCalculatorOptions::ext)
|
||||
.frame_rate(),
|
||||
@@ -623,13 +648,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetPacketResamplingOptions) {
|
||||
|
||||
TEST_F(UnpackMediaSequenceCalculatorTest, GetFrameRateFromExample) {
|
||||
SetUpCalculator({}, {"IMAGE_FRAME_RATE:frame_rate"});
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
|
||||
Adopt(sequence_.release());
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
MP_EXPECT_OK(runner_->OutputSidePackets()
|
||||
.Tag("IMAGE_FRAME_RATE")
|
||||
.Tag(kImageFrameRateTag)
|
||||
.ValidateAsType<double>());
|
||||
EXPECT_EQ(runner_->OutputSidePackets().Tag("IMAGE_FRAME_RATE").Get<double>(),
|
||||
EXPECT_EQ(runner_->OutputSidePackets().Tag(kImageFrameRateTag).Get<double>(),
|
||||
image_frame_rate_);
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,10 @@ namespace {
|
||||
|
||||
namespace tf = ::tensorflow;
|
||||
|
||||
constexpr char kSingleIntTag[] = "SINGLE_INT";
|
||||
constexpr char kTensorOutTag[] = "TENSOR_OUT";
|
||||
constexpr char kVectorIntTag[] = "VECTOR_INT";
|
||||
|
||||
class VectorIntToTensorCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUpRunner(
|
||||
@@ -61,13 +65,13 @@ class VectorIntToTensorCalculatorTest : public ::testing::Test {
|
||||
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()
|
||||
->Tag("VECTOR_INT")
|
||||
->Tag(kVectorIntTag)
|
||||
.packets.push_back(Adopt(input.release()).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("TENSOR_OUT").packets;
|
||||
runner_->Outputs().Tag(kTensorOutTag).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
|
||||
@@ -95,13 +99,13 @@ TEST_F(VectorIntToTensorCalculatorTest, TestSingleValue) {
|
||||
tensorflow::DT_INT32, false, true);
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()
|
||||
->Tag("SINGLE_INT")
|
||||
->Tag(kSingleIntTag)
|
||||
.packets.push_back(MakePacket<int>(1).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("TENSOR_OUT").packets;
|
||||
runner_->Outputs().Tag(kTensorOutTag).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
|
||||
@@ -121,13 +125,13 @@ TEST_F(VectorIntToTensorCalculatorTest, TesOneDim) {
|
||||
}
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()
|
||||
->Tag("VECTOR_INT")
|
||||
->Tag(kVectorIntTag)
|
||||
.packets.push_back(Adopt(input.release()).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("TENSOR_OUT").packets;
|
||||
runner_->Outputs().Tag(kTensorOutTag).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
|
||||
@@ -152,13 +156,13 @@ TEST_F(VectorIntToTensorCalculatorTest, TestInt64) {
|
||||
tensorflow::DT_INT64, false, true);
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()
|
||||
->Tag("SINGLE_INT")
|
||||
->Tag(kSingleIntTag)
|
||||
.packets.push_back(MakePacket<int>(1LL << 31).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("TENSOR_OUT").packets;
|
||||
runner_->Outputs().Tag(kTensorOutTag).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
|
||||
@@ -179,13 +183,13 @@ TEST_F(VectorIntToTensorCalculatorTest, TestUint8) {
|
||||
}
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()
|
||||
->Tag("VECTOR_INT")
|
||||
->Tag(kVectorIntTag)
|
||||
.packets.push_back(Adopt(input.release()).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag("TENSOR_OUT").packets;
|
||||
runner_->Outputs().Tag(kTensorOutTag).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
|
||||
|
||||
@@ -162,6 +162,27 @@ selects.config_setting_group(
|
||||
],
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "edge_tpu_usb",
|
||||
define_values = {
|
||||
"MEDIAPIPE_EDGE_TPU": "usb",
|
||||
},
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "edge_tpu_pci",
|
||||
define_values = {
|
||||
"MEDIAPIPE_EDGE_TPU": "pci",
|
||||
},
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "edge_tpu_all",
|
||||
define_values = {
|
||||
"MEDIAPIPE_EDGE_TPU": "all",
|
||||
},
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tflite_inference_calculator",
|
||||
srcs = ["tflite_inference_calculator.cc"],
|
||||
@@ -172,6 +193,12 @@ cc_library(
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
defines = select({
|
||||
"//conditions:default": [],
|
||||
":edge_tpu_usb": ["MEDIAPIPE_EDGE_TPU=usb"],
|
||||
":edge_tpu_pci": ["MEDIAPIPE_EDGE_TPU=pci"],
|
||||
":edge_tpu_all": ["MEDIAPIPE_EDGE_TPU=all"],
|
||||
}),
|
||||
linkopts = select({
|
||||
"//mediapipe:ios": [
|
||||
"-framework CoreVideo",
|
||||
@@ -223,6 +250,20 @@ cc_library(
|
||||
"//conditions:default": [
|
||||
"//mediapipe/util:cpu_util",
|
||||
],
|
||||
}) + select({
|
||||
"//conditions:default": [],
|
||||
":edge_tpu_usb": [
|
||||
"@libedgetpu//tflite/public:edgetpu",
|
||||
"@libedgetpu//tflite/public:oss_edgetpu_direct_usb",
|
||||
],
|
||||
":edge_tpu_pci": [
|
||||
"@libedgetpu//tflite/public:edgetpu",
|
||||
"@libedgetpu//tflite/public:oss_edgetpu_direct_pci",
|
||||
],
|
||||
":edge_tpu_all": [
|
||||
"@libedgetpu//tflite/public:edgetpu",
|
||||
"@libedgetpu//tflite/public:oss_edgetpu_direct_all",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
@@ -85,7 +85,22 @@ constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
|
||||
} // namespace
|
||||
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
#include "edgetpu.h"
|
||||
#include "tflite/public/edgetpu.h"
|
||||
|
||||
// Checkes whether model contains Edge TPU custom op or not.
|
||||
bool ContainsEdgeTpuCustomOp(const tflite::FlatBufferModel& model) {
|
||||
const auto* opcodes = model.GetModel()->operator_codes();
|
||||
for (const auto* subgraph : *model.GetModel()->subgraphs()) {
|
||||
for (const auto* op : *subgraph->operators()) {
|
||||
const auto* opcode = opcodes->Get(op->opcode_index());
|
||||
if (opcode->custom_code() &&
|
||||
opcode->custom_code()->str() == edgetpu::kCustomOp) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Creates and returns an Edge TPU interpreter to run the given edgetpu model.
|
||||
std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
|
||||
@@ -94,14 +109,9 @@ std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
|
||||
edgetpu::EdgeTpuContext* edgetpu_context) {
|
||||
resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp());
|
||||
std::unique_ptr<tflite::Interpreter> interpreter;
|
||||
if (tflite::InterpreterBuilder(model, *resolver)(&interpreter) != kTfLiteOk) {
|
||||
std::cerr << "Failed to build edge TPU interpreter." << std::endl;
|
||||
}
|
||||
CHECK_EQ(tflite::InterpreterBuilder(model, *resolver)(&interpreter),
|
||||
kTfLiteOk);
|
||||
interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context);
|
||||
interpreter->SetNumThreads(1);
|
||||
if (interpreter->AllocateTensors() != kTfLiteOk) {
|
||||
std::cerr << "Failed to allocate edge TPU tensors." << std::endl;
|
||||
}
|
||||
return interpreter;
|
||||
}
|
||||
#endif // MEDIAPIPE_EDGE_TPU
|
||||
@@ -279,8 +289,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
std::shared_ptr<edgetpu::EdgeTpuContext> edgetpu_context_ =
|
||||
edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice();
|
||||
std::shared_ptr<edgetpu::EdgeTpuContext> edgetpu_context_;
|
||||
#endif
|
||||
|
||||
bool gpu_inference_ = false;
|
||||
@@ -292,6 +301,8 @@ class TfLiteInferenceCalculator : public CalculatorBase {
|
||||
bool allow_precision_loss_ = false;
|
||||
mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::Api
|
||||
tflite_gpu_runner_api_;
|
||||
mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::InferenceUsage
|
||||
tflite_gpu_runner_usage_;
|
||||
|
||||
bool use_kernel_caching_ = false;
|
||||
std::string cached_kernel_filename_;
|
||||
@@ -301,6 +312,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
// Calculator Core Section
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kCustomOpResolverTag[] = "CUSTOM_OP_RESOLVER";
|
||||
constexpr char kModelTag[] = "MODEL";
|
||||
|
||||
template <class CC>
|
||||
bool ShouldUseGpu(CC* cc) {
|
||||
#if MEDIAPIPE_TFLITE_GPU_SUPPORTED
|
||||
@@ -325,7 +340,7 @@ absl::Status TfLiteInferenceCalculator::GetContract(CalculatorContract* cc) {
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
|
||||
RET_CHECK(!options.model_path().empty() ^
|
||||
cc->InputSidePackets().HasTag("MODEL"))
|
||||
cc->InputSidePackets().HasTag(kModelTag))
|
||||
<< "Either model as side packet or model path in options is required.";
|
||||
|
||||
if (cc->Inputs().HasTag(kTensorsTag))
|
||||
@@ -338,13 +353,13 @@ absl::Status TfLiteInferenceCalculator::GetContract(CalculatorContract* cc) {
|
||||
if (cc->Outputs().HasTag(kTensorsGpuTag))
|
||||
cc->Outputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
|
||||
|
||||
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
|
||||
if (cc->InputSidePackets().HasTag(kCustomOpResolverTag)) {
|
||||
cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Tag(kCustomOpResolverTag)
|
||||
.Set<tflite::ops::builtin::BuiltinOpResolver>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("MODEL")) {
|
||||
cc->InputSidePackets().Tag("MODEL").Set<TfLiteModelPtr>();
|
||||
if (cc->InputSidePackets().HasTag(kModelTag)) {
|
||||
cc->InputSidePackets().Tag(kModelTag).Set<TfLiteModelPtr>();
|
||||
}
|
||||
|
||||
if (ShouldUseGpu(cc)) {
|
||||
@@ -377,6 +392,7 @@ absl::Status TfLiteInferenceCalculator::Open(CalculatorContext* cc) {
|
||||
options.delegate().gpu().use_advanced_gpu_api();
|
||||
allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
|
||||
tflite_gpu_runner_api_ = options.delegate().gpu().api();
|
||||
tflite_gpu_runner_usage_ = options.delegate().gpu().usage();
|
||||
|
||||
use_kernel_caching_ = use_advanced_gpu_api_ &&
|
||||
options.delegate().gpu().has_cached_kernel_path();
|
||||
@@ -483,8 +499,8 @@ absl::Status TfLiteInferenceCalculator::Close(CalculatorContext* cc) {
|
||||
MP_RETURN_IF_ERROR(WriteKernelsToFile());
|
||||
|
||||
return RunInContextIfNeeded([this]() -> absl::Status {
|
||||
interpreter_ = nullptr;
|
||||
if (delegate_) {
|
||||
interpreter_ = nullptr;
|
||||
delegate_ = nullptr;
|
||||
#if MEDIAPIPE_TFLITE_GPU_SUPPORTED
|
||||
if (gpu_inference_) {
|
||||
@@ -498,7 +514,7 @@ absl::Status TfLiteInferenceCalculator::Close(CalculatorContext* cc) {
|
||||
#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED
|
||||
}
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
edgetpu_context_.reset();
|
||||
edgetpu_context_ = nullptr;
|
||||
#endif
|
||||
return absl::OkStatus();
|
||||
});
|
||||
@@ -720,9 +736,9 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
|
||||
auto op_resolver_ptr =
|
||||
static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
|
||||
&default_op_resolver);
|
||||
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
|
||||
if (cc->InputSidePackets().HasTag(kCustomOpResolverTag)) {
|
||||
op_resolver_ptr = &(cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Tag(kCustomOpResolverTag)
|
||||
.Get<tflite::ops::builtin::BuiltinOpResolver>());
|
||||
}
|
||||
|
||||
@@ -733,7 +749,23 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
|
||||
: tflite::gpu::InferencePriority::MAX_PRECISION;
|
||||
options.priority2 = tflite::gpu::InferencePriority::AUTO;
|
||||
options.priority3 = tflite::gpu::InferencePriority::AUTO;
|
||||
options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
|
||||
switch (tflite_gpu_runner_usage_) {
|
||||
case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::
|
||||
FAST_SINGLE_ANSWER: {
|
||||
options.usage = tflite::gpu::InferenceUsage::FAST_SINGLE_ANSWER;
|
||||
break;
|
||||
}
|
||||
case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::
|
||||
SUSTAINED_SPEED: {
|
||||
options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
|
||||
break;
|
||||
}
|
||||
case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::
|
||||
UNSPECIFIED: {
|
||||
return absl::InternalError("inference usage need to be specified.");
|
||||
}
|
||||
}
|
||||
|
||||
tflite_gpu_runner_ = std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
|
||||
switch (tflite_gpu_runner_api_) {
|
||||
case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::OPENGL: {
|
||||
@@ -806,21 +838,26 @@ absl::Status TfLiteInferenceCalculator::LoadModel(CalculatorContext* cc) {
|
||||
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates
|
||||
default_op_resolver;
|
||||
auto op_resolver_ptr =
|
||||
static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
|
||||
&default_op_resolver);
|
||||
|
||||
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
|
||||
op_resolver_ptr = &(cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Get<tflite::ops::builtin::BuiltinOpResolver>());
|
||||
}
|
||||
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
interpreter_ =
|
||||
BuildEdgeTpuInterpreter(model, op_resolver_ptr, edgetpu_context_.get());
|
||||
#else
|
||||
tflite::InterpreterBuilder(model, *op_resolver_ptr)(&interpreter_);
|
||||
if (ContainsEdgeTpuCustomOp(model)) {
|
||||
edgetpu_context_ = edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice();
|
||||
interpreter_ = BuildEdgeTpuInterpreter(model, &default_op_resolver,
|
||||
edgetpu_context_.get());
|
||||
} else {
|
||||
#endif // MEDIAPIPE_EDGE_TPU
|
||||
auto op_resolver_ptr =
|
||||
static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
|
||||
&default_op_resolver);
|
||||
|
||||
if (cc->InputSidePackets().HasTag(kCustomOpResolverTag)) {
|
||||
op_resolver_ptr = &(cc->InputSidePackets()
|
||||
.Tag(kCustomOpResolverTag)
|
||||
.Get<tflite::ops::builtin::BuiltinOpResolver>());
|
||||
}
|
||||
|
||||
tflite::InterpreterBuilder(model, *op_resolver_ptr)(&interpreter_);
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
}
|
||||
#endif // MEDIAPIPE_EDGE_TPU
|
||||
|
||||
RET_CHECK(interpreter_);
|
||||
@@ -853,8 +890,8 @@ absl::StatusOr<Packet> TfLiteInferenceCalculator::GetModelAsPacket(
|
||||
if (!options.model_path().empty()) {
|
||||
return TfLiteModelLoader::LoadFromPath(options.model_path());
|
||||
}
|
||||
if (cc.InputSidePackets().HasTag("MODEL")) {
|
||||
return cc.InputSidePackets().Tag("MODEL");
|
||||
if (cc.InputSidePackets().HasTag(kModelTag)) {
|
||||
return cc.InputSidePackets().Tag(kModelTag);
|
||||
}
|
||||
return absl::Status(absl::StatusCode::kNotFound,
|
||||
"Must specify TFLite model as path or loaded model.");
|
||||
@@ -878,11 +915,15 @@ absl::Status TfLiteInferenceCalculator::LoadDelegate(CalculatorContext* cc) {
|
||||
// Attempt to use NNAPI.
|
||||
// If not supported, the default CPU delegate will be created and used.
|
||||
interpreter_->SetAllowFp16PrecisionForFp32(1);
|
||||
delegate_ =
|
||||
TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
|
||||
// No need to free according to tflite::NnApiDelegate()
|
||||
// documentation.
|
||||
});
|
||||
tflite::StatefulNnApiDelegate::Options options;
|
||||
const auto& nnapi = calculator_opts.delegate().nnapi();
|
||||
// Set up cache_dir and model_token for NNAPI compilation cache.
|
||||
if (nnapi.has_cache_dir() && nnapi.has_model_token()) {
|
||||
options.cache_dir = nnapi.cache_dir().c_str();
|
||||
options.model_token = nnapi.model_token().c_str();
|
||||
}
|
||||
delegate_ = TfLiteDelegatePtr(new tflite::StatefulNnApiDelegate(options),
|
||||
[](TfLiteDelegate*) {});
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
|
||||
kTfLiteOk);
|
||||
return absl::OkStatus();
|
||||
@@ -906,6 +947,8 @@ absl::Status TfLiteInferenceCalculator::LoadDelegate(CalculatorContext* cc) {
|
||||
kTfLiteOk);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
#else
|
||||
(void)use_xnnpack;
|
||||
#endif // !EDGETPU
|
||||
|
||||
// Return and use default tflite infernece (on CPU). No need for GPU
|
||||
|
||||
@@ -67,9 +67,31 @@ message TfLiteInferenceCalculatorOptions {
|
||||
// Only available for OpenCL delegate on Android.
|
||||
// Kernel caching will only be enabled if this path is set.
|
||||
optional string cached_kernel_path = 2;
|
||||
|
||||
// Encapsulated compilation/runtime tradeoffs.
|
||||
enum InferenceUsage {
|
||||
UNSPECIFIED = 0;
|
||||
|
||||
// InferenceRunner will be used only once. Therefore, it is important to
|
||||
// minimize bootstrap time as well.
|
||||
FAST_SINGLE_ANSWER = 1;
|
||||
|
||||
// Prefer maximizing the throughput. Same inference runner will be used
|
||||
// repeatedly on different inputs.
|
||||
SUSTAINED_SPEED = 2;
|
||||
}
|
||||
optional InferenceUsage usage = 5 [default = SUSTAINED_SPEED];
|
||||
}
|
||||
// Android only.
|
||||
message Nnapi {}
|
||||
message Nnapi {
|
||||
// Directory to store compilation cache. If unspecified, NNAPI will not
|
||||
// try caching the compilation.
|
||||
optional string cache_dir = 1;
|
||||
// Unique token identifying the model. It is the caller's responsibility
|
||||
// to ensure there is no clash of the tokens. If unspecified, NNAPI will
|
||||
// not try caching the compilation.
|
||||
optional string model_token = 2;
|
||||
}
|
||||
message Xnnpack {
|
||||
// Number of threads for XNNPACK delegate. (By default, calculator tries
|
||||
// to choose optimal number of threads depending on the device.)
|
||||
|
||||
@@ -18,6 +18,10 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kFloatsTag[] = "FLOATS";
|
||||
constexpr char kFloatTag[] = "FLOAT";
|
||||
constexpr char kTensorsTag[] = "TENSORS";
|
||||
|
||||
// A calculator for converting TFLite tensors to to a float or a float vector.
|
||||
//
|
||||
// Input:
|
||||
@@ -48,15 +52,16 @@ REGISTER_CALCULATOR(TfLiteTensorsToFloatsCalculator);
|
||||
|
||||
absl::Status TfLiteTensorsToFloatsCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag("TENSORS"));
|
||||
RET_CHECK(cc->Outputs().HasTag("FLOATS") || cc->Outputs().HasTag("FLOAT"));
|
||||
RET_CHECK(cc->Inputs().HasTag(kTensorsTag));
|
||||
RET_CHECK(cc->Outputs().HasTag(kFloatsTag) ||
|
||||
cc->Outputs().HasTag(kFloatTag));
|
||||
|
||||
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
|
||||
if (cc->Outputs().HasTag("FLOATS")) {
|
||||
cc->Outputs().Tag("FLOATS").Set<std::vector<float>>();
|
||||
cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
|
||||
if (cc->Outputs().HasTag(kFloatsTag)) {
|
||||
cc->Outputs().Tag(kFloatsTag).Set<std::vector<float>>();
|
||||
}
|
||||
if (cc->Outputs().HasTag("FLOAT")) {
|
||||
cc->Outputs().Tag("FLOAT").Set<float>();
|
||||
if (cc->Outputs().HasTag(kFloatTag)) {
|
||||
cc->Outputs().Tag(kFloatTag).Set<float>();
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
@@ -69,10 +74,10 @@ absl::Status TfLiteTensorsToFloatsCalculator::Open(CalculatorContext* cc) {
|
||||
}
|
||||
|
||||
absl::Status TfLiteTensorsToFloatsCalculator::Process(CalculatorContext* cc) {
|
||||
RET_CHECK(!cc->Inputs().Tag("TENSORS").IsEmpty());
|
||||
RET_CHECK(!cc->Inputs().Tag(kTensorsTag).IsEmpty());
|
||||
|
||||
const auto& input_tensors =
|
||||
cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
|
||||
cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
|
||||
// TODO: Add option to specify which tensor to take from.
|
||||
const TfLiteTensor* raw_tensor = &input_tensors[0];
|
||||
const float* raw_floats = raw_tensor->data.f;
|
||||
@@ -82,18 +87,19 @@ absl::Status TfLiteTensorsToFloatsCalculator::Process(CalculatorContext* cc) {
|
||||
num_values *= raw_tensor->dims->data[i];
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag("FLOAT")) {
|
||||
if (cc->Outputs().HasTag(kFloatTag)) {
|
||||
// TODO: Could add an index in the option to specifiy returning one
|
||||
// value of a float array.
|
||||
RET_CHECK_EQ(num_values, 1);
|
||||
cc->Outputs().Tag("FLOAT").AddPacket(
|
||||
cc->Outputs().Tag(kFloatTag).AddPacket(
|
||||
MakePacket<float>(raw_floats[0]).At(cc->InputTimestamp()));
|
||||
}
|
||||
if (cc->Outputs().HasTag("FLOATS")) {
|
||||
if (cc->Outputs().HasTag(kFloatsTag)) {
|
||||
auto output_floats = absl::make_unique<std::vector<float>>(
|
||||
raw_floats, raw_floats + num_values);
|
||||
cc->Outputs().Tag("FLOATS").Add(output_floats.release(),
|
||||
cc->InputTimestamp());
|
||||
cc->Outputs()
|
||||
.Tag(kFloatsTag)
|
||||
.Add(output_floats.release(), cc->InputTimestamp());
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
|
||||
@@ -1353,3 +1353,34 @@ cc_test(
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "inverse_matrix_calculator",
|
||||
srcs = ["inverse_matrix_calculator.cc"],
|
||||
hdrs = ["inverse_matrix_calculator.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/api2:port",
|
||||
"@com_google_absl//absl/status",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = True,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "inverse_matrix_calculator_test",
|
||||
srcs = ["inverse_matrix_calculator_test.cc"],
|
||||
tags = ["desktop_only_test"],
|
||||
deps = [
|
||||
":inverse_matrix_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -87,7 +87,7 @@ absl::Status ClockTimestampCalculator::Open(CalculatorContext* cc) {
|
||||
// Initialize the clock.
|
||||
if (cc->InputSidePackets().HasTag(kClockTag)) {
|
||||
clock_ = cc->InputSidePackets()
|
||||
.Tag("CLOCK")
|
||||
.Tag(kClockTag)
|
||||
.Get<std::shared_ptr<::mediapipe::Clock>>();
|
||||
} else {
|
||||
clock_.reset(
|
||||
|
||||
@@ -27,6 +27,8 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
constexpr char kIterableTag[] = "ITERABLE";
|
||||
|
||||
typedef CollectionHasMinSizeCalculator<std::vector<int>>
|
||||
TestIntCollectionHasMinSizeCalculator;
|
||||
REGISTER_CALCULATOR(TestIntCollectionHasMinSizeCalculator);
|
||||
@@ -34,7 +36,7 @@ REGISTER_CALCULATOR(TestIntCollectionHasMinSizeCalculator);
|
||||
void AddInputVector(const std::vector<int>& input, int64 timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
runner->MutableInputs()
|
||||
->Tag("ITERABLE")
|
||||
->Tag(kIterableTag)
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
|
||||
}
|
||||
|
||||
@@ -144,7 +144,7 @@ class DetectionLetterboxRemovalCalculator : public CalculatorBase {
|
||||
}
|
||||
|
||||
cc->Outputs()
|
||||
.Tag("DETECTIONS")
|
||||
.Tag(kDetectionsTag)
|
||||
.Add(output_detections.release(), cc->InputTimestamp());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user