Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 MediaPipe Teamandchuoling
|
ecb5b5f44a | ||
|
MediaPipe Teamandchuoling
|
7c331ad58b | ||
|
MediaPipe Teamandchuoling
|
a92cff7a60 |
+1
-1
@@ -54,7 +54,7 @@ RUN pip3 install tf_slim
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# Install bazel
|
||||
ARG BAZEL_VERSION=3.4.1
|
||||
ARG BAZEL_VERSION=3.7.2
|
||||
RUN mkdir /bazel && \
|
||||
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
|
||||
azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||
|
||||
@@ -10,3 +10,7 @@ include requirements.txt
|
||||
recursive-include mediapipe/modules *.tflite *.txt *.binarypb
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_sneakers.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_chair.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_camera.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_cup.tflite
|
||||
|
||||
@@ -44,7 +44,7 @@ Hair Segmentation
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
|
||||
@@ -2,16 +2,19 @@ workspace(name = "mediapipe")
|
||||
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
skylib_version = "0.9.0"
|
||||
http_archive(
|
||||
name = "bazel_skylib",
|
||||
type = "tar.gz",
|
||||
url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
|
||||
sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
|
||||
urls = [
|
||||
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
|
||||
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
|
||||
],
|
||||
sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
|
||||
)
|
||||
load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
|
||||
bazel_skylib_workspace()
|
||||
load("@bazel_skylib//lib:versions.bzl", "versions")
|
||||
versions.check(minimum_bazel_version = "3.4.0")
|
||||
|
||||
versions.check(minimum_bazel_version = "3.7.2")
|
||||
|
||||
# ABSL cpp library lts_2020_09_23
|
||||
http_archive(
|
||||
@@ -38,8 +41,8 @@ http_archive(
|
||||
|
||||
http_archive(
|
||||
name = "rules_foreign_cc",
|
||||
strip_prefix = "rules_foreign_cc-main",
|
||||
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/main.zip",
|
||||
strip_prefix = "rules_foreign_cc-0.1.0",
|
||||
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
|
||||
)
|
||||
|
||||
load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
|
||||
@@ -117,7 +120,8 @@ http_archive(
|
||||
# libyuv
|
||||
http_archive(
|
||||
name = "libyuv",
|
||||
urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"],
|
||||
# Error: operand type mismatch for `vbroadcastss' caused by commit 8a13626e42f7fdcf3a6acbb0316760ee54cda7d8.
|
||||
urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/2525698acba9bf9b701ba6b4d9584291a1f62257.tar.gz"],
|
||||
build_file = "@//third_party:libyuv.BUILD",
|
||||
)
|
||||
|
||||
@@ -304,8 +308,8 @@ http_archive(
|
||||
|
||||
# Maven dependencies.
|
||||
|
||||
RULES_JVM_EXTERNAL_TAG = "3.2"
|
||||
RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
|
||||
RULES_JVM_EXTERNAL_TAG = "4.0"
|
||||
RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169"
|
||||
|
||||
http_archive(
|
||||
name = "rules_jvm_external",
|
||||
@@ -318,7 +322,6 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
|
||||
|
||||
# Important: there can only be one maven_install rule. Add new maven deps here.
|
||||
maven_install(
|
||||
name = "maven",
|
||||
artifacts = [
|
||||
"androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
"androidx.lifecycle:lifecycle-common:2.2.0",
|
||||
@@ -343,10 +346,10 @@ maven_install(
|
||||
"org.hamcrest:hamcrest-library:1.3",
|
||||
],
|
||||
repositories = [
|
||||
"https://jcenter.bintray.com",
|
||||
"https://maven.google.com",
|
||||
"https://dl.google.com/dl/android/maven2",
|
||||
"https://repo1.maven.org/maven2",
|
||||
"https://jcenter.bintray.com",
|
||||
],
|
||||
fetch_sources = True,
|
||||
version_conflict_policy = "pinned",
|
||||
@@ -363,10 +366,10 @@ http_archive(
|
||||
],
|
||||
)
|
||||
|
||||
#Tensorflow repo should always go after the other external dependencies.
|
||||
# 2020-12-09
|
||||
_TENSORFLOW_GIT_COMMIT = "0eadbb13cef1226b1bae17c941f7870734d97f8a"
|
||||
_TENSORFLOW_SHA256= "4ae06daa5b09c62f31b7bc1f781fd59053f286dd64355830d8c2ac601b795ef0"
|
||||
# Tensorflow repo should always go after the other external dependencies.
|
||||
# 2021-03-25
|
||||
_TENSORFLOW_GIT_COMMIT = "c67f68021824410ebe9f18513b8856ac1c6d4887"
|
||||
_TENSORFLOW_SHA256= "fd07d0b39422dc435e268c5e53b2646a8b4b1e3151b87837b43f86068faae87f"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
urls = [
|
||||
@@ -383,5 +386,7 @@ http_archive(
|
||||
sha256 = _TENSORFLOW_SHA256,
|
||||
)
|
||||
|
||||
load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
|
||||
tf_workspace(tf_repo_name = "org_tensorflow")
|
||||
load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
|
||||
tf_workspace3()
|
||||
load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
|
||||
tf_workspace2()
|
||||
|
||||
@@ -12,19 +12,30 @@ nav_order: 3
|
||||
{:toc}
|
||||
---
|
||||
|
||||
Each calculator is a node of of a graph. We describe how to create a new calculator, how to initialize a calculator, how to perform its calculations, input and output streams, timestamps, and options
|
||||
Calculators communicate by sending and receiving packets. Typically a single
|
||||
packet is sent along each input stream at each input timestamp. A packet can
|
||||
contain any kind of data, such as a single frame of video or a single integer
|
||||
detection count.
|
||||
|
||||
## Creating a packet
|
||||
|
||||
Packets are generally created with `MediaPipe::Adopt()` (from packet.h).
|
||||
Packets are generally created with `mediapipe::MakePacket<T>()` or
|
||||
`mediapipe::Adopt()` (from packet.h).
|
||||
|
||||
```c++
|
||||
// Create some data.
|
||||
auto data = absl::make_unique<MyDataClass>("constructor_argument");
|
||||
// Create a packet to own the data.
|
||||
Packet p = Adopt(data.release());
|
||||
// Create a packet containing some new data.
|
||||
Packet p = MakePacket<MyDataClass>("constructor_argument");
|
||||
// Make a new packet with the same data and a different timestamp.
|
||||
Packet p2 = p.At(Timestamp::PostStream());
|
||||
```
|
||||
|
||||
or:
|
||||
|
||||
```c++
|
||||
// Create some new data.
|
||||
auto data = absl::make_unique<MyDataClass>("constructor_argument");
|
||||
// Create a packet to own the data.
|
||||
Packet p = Adopt(data.release()).At(Timestamp::PostStream());
|
||||
```
|
||||
|
||||
Data within a packet is accessed with `Packet::Get<T>()`
|
||||
|
||||
@@ -28,7 +28,7 @@ Gradle.
|
||||
* Install MediaPipe following these [instructions](./install.md).
|
||||
* Setup Java Runtime.
|
||||
* Setup Android SDK release 28.0.3 and above.
|
||||
* Setup Android NDK r18b and above.
|
||||
* Setup Android NDK version between 18 and 21.
|
||||
|
||||
MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
|
||||
below for Android Studio setup). However, if you prefer using MediaPipe without
|
||||
|
||||
@@ -25,25 +25,11 @@ install --user six`.
|
||||
|
||||
## Installing on Debian and Ubuntu
|
||||
|
||||
1. Install Bazel.
|
||||
1. Install Bazelisk.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
For Nvidia Jetson and Raspberry Pi devices with aarch64 Linux, Bazel needs
|
||||
to be built from source:
|
||||
|
||||
```bash
|
||||
# For Bazel 3.4.1
|
||||
mkdir $HOME/bazel-3.4.1
|
||||
cd $HOME/bazel-3.4.1
|
||||
wget https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-dist.zip
|
||||
sudo apt-get install build-essential openjdk-8-jdk python zip unzip
|
||||
unzip bazel-3.4.1-dist.zip
|
||||
env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
|
||||
sudo cp output/bazel /usr/local/bin/
|
||||
```
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
2. Checkout MediaPipe repository.
|
||||
|
||||
@@ -207,11 +193,11 @@ build issues.
|
||||
|
||||
**Disclaimer**: Running MediaPipe on CentOS is experimental.
|
||||
|
||||
1. Install Bazel.
|
||||
1. Install Bazelisk.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
|
||||
to install Bazel 3.4 or higher.
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
2. Checkout MediaPipe repository.
|
||||
|
||||
@@ -336,11 +322,11 @@ build issues.
|
||||
* Install [Xcode](https://developer.apple.com/xcode/) and its Command Line
|
||||
Tools by `xcode-select --install`.
|
||||
|
||||
2. Install Bazel.
|
||||
2. Install Bazelisk.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
|
||||
to install Bazel 3.4 or higher.
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
3. Checkout MediaPipe repository.
|
||||
|
||||
@@ -353,7 +339,7 @@ build issues.
|
||||
4. Install OpenCV and FFmpeg.
|
||||
|
||||
Option 1. Use HomeBrew package manager tool to install the pre-compiled
|
||||
OpenCV 3.4.5 libraries. FFmpeg will be installed via OpenCV.
|
||||
OpenCV 3 libraries. FFmpeg will be installed via OpenCV.
|
||||
|
||||
```bash
|
||||
$ brew install opencv@3
|
||||
@@ -484,29 +470,36 @@ next section.
|
||||
|
||||
4. Install Visual C++ Build Tools 2019 and WinSDK
|
||||
|
||||
Go to https://visualstudio.microsoft.com/visual-cpp-build-tools, download
|
||||
build tools, and install Microsoft Visual C++ 2019 Redistributable and
|
||||
Microsoft Build Tools 2019.
|
||||
Go to
|
||||
[the VisualStudio website](ttps://visualstudio.microsoft.com/visual-cpp-build-tools),
|
||||
download build tools, and install Microsoft Visual C++ 2019 Redistributable
|
||||
and Microsoft Build Tools 2019.
|
||||
|
||||
Download the WinSDK from
|
||||
https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/ and
|
||||
install.
|
||||
[the official MicroSoft website](https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/)
|
||||
and install.
|
||||
|
||||
5. Install Bazel and add the location of the Bazel executable to the `%PATH%`
|
||||
environment variable.
|
||||
5. Install Bazel or Bazelisk and add the location of the Bazel executable to
|
||||
the `%PATH%` environment variable.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
|
||||
to install Bazel 3.4 or higher.
|
||||
Option 1. Follow
|
||||
[the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
|
||||
to install Bazel 3.7.2 or higher.
|
||||
|
||||
6. Set Bazel variables.
|
||||
Option 2. Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
6. Set Bazel variables. Learn more details about
|
||||
["Build on Windows"](https://docs.bazel.build/versions/master/windows.html#build-c-with-msvc)
|
||||
in the Bazel official documentation.
|
||||
|
||||
```
|
||||
# Find the exact paths and version numbers from your local version.
|
||||
# Please find the exact paths and version numbers from your local version.
|
||||
C:\> set BAZEL_VS=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools
|
||||
C:\> set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC
|
||||
C:\> set BAZEL_VC_FULL_VERSION=14.25.28610
|
||||
C:\> set BAZEL_WINSDK_FULL_VERSION=10.1.18362.1
|
||||
C:\> set BAZEL_VC_FULL_VERSION=<Your local VC version>
|
||||
C:\> set BAZEL_WINSDK_FULL_VERSION=<Your local WinSDK version>
|
||||
```
|
||||
|
||||
7. Checkout MediaPipe repository.
|
||||
@@ -593,19 +586,11 @@ cameras. Alternatively, you use a video file as input.
|
||||
username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
|
||||
```
|
||||
|
||||
5. Install Bazel.
|
||||
5. Install Bazelisk.
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
|
||||
https://storage.googleapis.com/bazel/3.4.1/release/bazel-3.4.1-installer-linux-x86_64.sh && \
|
||||
sudo mkdir -p /usr/local/bazel/3.4.1 && \
|
||||
chmod 755 bazel-3.4.1-installer-linux-x86_64.sh && \
|
||||
sudo ./bazel-3.4.1-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.4.1 && \
|
||||
source /usr/local/bazel/3.4.1/lib/bazel/bin/bazel-complete.bash
|
||||
|
||||
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.4.1/lib/bazel/bin/bazel version && \
|
||||
alias bazel='/usr/local/bazel/3.4.1/lib/bazel/bin/bazel'
|
||||
```
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
6. Checkout MediaPipe repository.
|
||||
|
||||
|
||||
@@ -72,9 +72,9 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
|
||||
[Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection
|
||||
[H-npm]: https://www.npmjs.com/package/@mediapipe/hands
|
||||
[P-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[draw-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[cam-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[draw-npm]: https://www.npmjs.com/package/@mediapipe/drawing_utils
|
||||
[cam-npm]: https://www.npmjs.com/package/@mediapipe/camera_utils
|
||||
[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/control_utils
|
||||
[Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic
|
||||
[F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh
|
||||
[Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ Hair Segmentation
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
|
||||
@@ -78,6 +78,7 @@ Supported configuration options:
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_face_detction = mp.solutions.face_detection
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
|
||||
# For static images:
|
||||
with mp_face_detection.FaceDetection(
|
||||
@@ -183,8 +184,8 @@ function onResults(results) {
|
||||
canvasCtx.restore();
|
||||
}
|
||||
|
||||
const faceDetection = new Objectron({locateFile: (file) => {
|
||||
return `https://cdn.jsdelivr.net/npm/@mediapipe/objectr[email protected]/${file}`;
|
||||
const faceDetection = new FaceDetection({locateFile: (file) => {
|
||||
return `https://cdn.jsdelivr.net/npm/@mediapipe/face_detecti[email protected]/${file}`;
|
||||
}});
|
||||
faceDetection.setOptions({
|
||||
minDetectionConfidence: 0.5
|
||||
|
||||
+40
-11
@@ -358,15 +358,17 @@ cap.release()
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/android.md) and [iOS](../getting_started/ios.md) on
|
||||
how to build MediaPipe examples.
|
||||
[Android](../getting_started/android.md), [iOS](../getting_started/ios.md), and
|
||||
[desktop](../getting_started/cpp.md) on how to build MediaPipe examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
### Two-stage Objectron
|
||||
### Mobile
|
||||
|
||||
#### Two-stage Objectron
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
|
||||
@@ -404,7 +406,7 @@ to visualize its associated subgraphs, please see
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
### Single-stage Objectron
|
||||
#### Single-stage Objectron
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
|
||||
@@ -428,7 +430,7 @@ to visualize its associated subgraphs, please see
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
### Assets
|
||||
#### Assets
|
||||
|
||||
Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc) using a parsing of the sequenced .obj file
|
||||
format into a custom .uuu format. This can be done for user assets as follows:
|
||||
@@ -449,9 +451,35 @@ Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](http
|
||||
> single .uuu animation file, using the order given by sorting the filenames alphanumerically. Also the ObjParser directory inputs must be given as
|
||||
> absolute paths, not relative paths. See parser utility library at [`mediapipe/graphs/object_detection_3d/obj_parser/`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/obj_parser/) for more details.
|
||||
|
||||
### Coordinate Systems
|
||||
|
||||
#### Object Coordinate
|
||||
### Desktop
|
||||
|
||||
To build the application, run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/object_detection_3d:objectron_cpu
|
||||
```
|
||||
|
||||
To run the application, replace `<input video path>` and `<output video path>`
|
||||
in the command below with your own paths, and `<landmark model path>` and
|
||||
`<allowed labels>` with the following:
|
||||
|
||||
Category | `<landmark model path>` | `<allowed labels>`
|
||||
:------- | :-------------------------------------------------------------------------- | :-----------------
|
||||
Shoe | mediapipe/modules/objectron/object_detection_3d_sneakers.tflite | Footwear
|
||||
Chair | mediapipe/modules/objectron/object_detection_3d_chair.tflite | Chair
|
||||
Cup | mediapipe/modules/objectron/object_detection_3d_cup.tflite | Mug
|
||||
Camera | mediapipe/modules/objectron/object_detection_3d_camera.tflite | Camera
|
||||
|
||||
```
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection_3d/objectron_cpu \
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt \
|
||||
--input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>,box_landmark_model_path=<landmark model path>,allowed_labels=<allowed labels>
|
||||
```
|
||||
|
||||
## Coordinate Systems
|
||||
|
||||
### Object Coordinate
|
||||
|
||||
Each object has its object coordinate frame. We use the below object coordinate
|
||||
definition, with `+x` pointing right, `+y` pointing up and `+z` pointing front,
|
||||
@@ -459,7 +487,7 @@ origin is at the center of the 3D bounding box.
|
||||
|
||||
|
||||
|
||||
#### Camera Coordinate
|
||||
### Camera Coordinate
|
||||
|
||||
A 3D object is parameterized by its `scale` and `rotation`, `translation` with
|
||||
regard to the camera coordinate frame. In this API we use the below camera
|
||||
@@ -476,7 +504,7 @@ camera frame by applying `rotation` and `translation`:
|
||||
landmarks_3d = rotation * scale * unit_box + translation
|
||||
```
|
||||
|
||||
#### NDC Space
|
||||
### NDC Space
|
||||
|
||||
In this API we use
|
||||
[NDC(normalized device coordinates)](http://www.songho.ca/opengl/gl_projectionmatrix.html)
|
||||
@@ -495,7 +523,7 @@ y_ndc = -fy * Y / Z + py
|
||||
z_ndc = 1 / Z
|
||||
```
|
||||
|
||||
#### Pixel Space
|
||||
### Pixel Space
|
||||
|
||||
In this API we set upper-left coner of an image as the origin of pixel
|
||||
coordinate. One can convert from NDC to pixel space as follows:
|
||||
@@ -532,10 +560,11 @@ py = -py_pixel * 2.0 / image_height + 1.0
|
||||
[Announcing the Objectron Dataset](https://ai.googleblog.com/2020/11/announcing-objectron-dataset.html)
|
||||
* Google AI Blog:
|
||||
[Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
|
||||
* Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in CVPR 2021
|
||||
* Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
|
||||
Shape Supervision](https://arxiv.org/abs/2003.03522)
|
||||
* Paper:
|
||||
[Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
|
||||
([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
|
||||
([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth Workshop on Computer Vision for AR/VR, CVPR 2020
|
||||
* [Models and model cards](./models.md#objectron)
|
||||
* [Python Colab](https://mediapipe.page.link/objectron_py_colab)
|
||||
|
||||
+5
-121
@@ -2,6 +2,8 @@
|
||||
layout: default
|
||||
title: Pose
|
||||
parent: Solutions
|
||||
has_children: true
|
||||
has_toc: false
|
||||
nav_order: 5
|
||||
---
|
||||
|
||||
@@ -21,10 +23,9 @@ nav_order: 5
|
||||
## Overview
|
||||
|
||||
Human pose estimation from video plays a critical role in various applications
|
||||
such as
|
||||
[quantifying physical exercises](#pose-classification-and-repetition-counting),
|
||||
sign language recognition, and full-body gesture control. For example, it can
|
||||
form the basis for yoga, dance, and fitness applications. It can also enable the
|
||||
such as [quantifying physical exercises](./pose_classification.md), sign
|
||||
language recognition, and full-body gesture control. For example, it can form
|
||||
the basis for yoga, dance, and fitness applications. It can also enable the
|
||||
overlay of digital content and information on top of the physical world in
|
||||
augmented reality.
|
||||
|
||||
@@ -387,121 +388,6 @@ on how to build MediaPipe examples.
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
|
||||
|
||||
## Pose Classification and Repetition Counting
|
||||
|
||||
One of the applications
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
can enable is fitness. More specifically - pose classification and repetition
|
||||
counting. In this section we'll provide basic guidance on building a custom pose
|
||||
classifier with the help of a
|
||||
[Colab](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
|
||||
and wrap it in a simple
|
||||
[fitness app](https://mediapipe.page.link/mlkit-pose-classification-demo-app)
|
||||
powered by [ML Kit](https://developers.google.com/ml-kit). Push-ups and squats
|
||||
are used for demonstration purposes as the most common exercises.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 4. Pose classification and repetition counting with MediaPipe Pose.* |
|
||||
|
||||
We picked the
|
||||
[k-nearest neighbors algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm)
|
||||
(k-NN) as the classifier. It's simple and easy to start with. The algorithm
|
||||
determines the object's class based on the closest samples in the training set.
|
||||
To build it, one needs to:
|
||||
|
||||
* Collect image samples of the target exercises and run pose prediction on
|
||||
them,
|
||||
* Convert obtained pose landmarks to a representation suitable for the k-NN
|
||||
classifier and form a training set,
|
||||
* Perform the classification itself followed by repetition counting.
|
||||
|
||||
### Training Set
|
||||
|
||||
To build a good classifier appropriate samples should be collected for the
|
||||
training set: about a few hundred samples for each terminal state of each
|
||||
exercise (e.g., "up" and "down" positions for push-ups). It's important that
|
||||
collected samples cover different camera angles, environment conditions, body
|
||||
shapes, and exercise variations.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 5. Two terminal states of push-ups.* |
|
||||
|
||||
To transform samples into a k-NN classifier training set, either
|
||||
[basic](https://drive.google.com/file/d/1z4IM8kG6ipHN6keadjD-F6vMiIIgViKK/view?usp=sharing)
|
||||
or
|
||||
[extended](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
|
||||
Colab could be used. They both use the
|
||||
[Python Solution API](#python-solution-api) to run the BlazePose models on given
|
||||
images and dump predicted pose landmarks to a CSV file. Additionally, the
|
||||
extended Colab provides useful tools to find outliers (e.g., wrongly predicted
|
||||
poses) and underrepresented classes (e.g., not covering all camera angles) by
|
||||
classifying each sample against the entire training set. After that, you'll be
|
||||
able to test the classifier on an arbitrary video right in the Colab.
|
||||
|
||||
### Classification
|
||||
|
||||
Code of the classifier is available both in the
|
||||
[extended](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
|
||||
Colab and in the
|
||||
[ML Kit demo app](https://mediapipe.page.link/mlkit-pose-classification-demo-app).
|
||||
Please refer to them for details of the approach described below.
|
||||
|
||||
The k-NN algorithm used for pose classification requires a feature vector
|
||||
representation of each sample and a metric to compute the distance between two
|
||||
such vectors to find the nearest pose samples to a target one.
|
||||
|
||||
To convert pose landmarks to a feature vector, we use pairwise distances between
|
||||
predefined lists of pose joints, such as distances between wrist and shoulder,
|
||||
ankle and hip, and two wrists. Since the algorithm relies on distances, all
|
||||
poses are normalized to have the same torso size and vertical torso orientation
|
||||
before the conversion.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 6. Main pairwise distances used for the pose feature vector.* |
|
||||
|
||||
To get a better classification result, k-NN search is invoked twice with
|
||||
different distance metrics:
|
||||
|
||||
* First, to filter out samples that are almost the same as the target one but
|
||||
have only a few different values in the feature vector (which means
|
||||
differently bent joints and thus other pose class), minimum per-coordinate
|
||||
distance is used as distance metric,
|
||||
* Then average per-coordinate distance is used to find the nearest pose
|
||||
cluster among those from the first search.
|
||||
|
||||
Finally, we apply
|
||||
[exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average)
|
||||
(EMA) smoothing to level any noise from pose prediction or classification. To do
|
||||
that, we search not only for the nearest pose cluster, but we calculate a
|
||||
probability for each of them and use it for smoothing over time.
|
||||
|
||||
### Repetition Counter
|
||||
|
||||
To count the repetitions, the algorithm monitors the probability of a target
|
||||
pose class. Let's take push-ups with its "up" and "down" terminal states:
|
||||
|
||||
* When the probability of the "down" pose class passes a certain threshold for
|
||||
the first time, the algorithm marks that the "down" pose class is entered.
|
||||
* Once the probability drops below the threshold, the algorithm marks that the
|
||||
"down" pose class has been exited and increases the counter.
|
||||
|
||||
To avoid cases when the probability fluctuates around the threshold (e.g., when
|
||||
the user pauses between "up" and "down" states) causing phantom counts, the
|
||||
threshold used to detect when the state is exited is actually slightly lower
|
||||
than the one used to detect when the state is entered. It creates an interval
|
||||
where the pose class and the counter can't be changed.
|
||||
|
||||
### Future Work
|
||||
|
||||
We are actively working on improving BlazePose GHUM 3D's Z prediction. It will
|
||||
allow us to use joint angles in the feature vectors, which are more natural and
|
||||
easier to configure (although distances can still be useful to detect touches
|
||||
between body parts) and to perform rotation normalization of poses and reduce
|
||||
the number of camera angles required for accurate k-NN classification.
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
@@ -512,5 +398,3 @@ the number of camera angles required for accurate k-NN classification.
|
||||
* [Models and model cards](./models.md#pose)
|
||||
* [Web demo](https://code.mediapipe.dev/codepen/pose)
|
||||
* [Python Colab](https://mediapipe.page.link/pose_py_colab)
|
||||
* [Pose Classification Colab (Basic)](https://mediapipe.page.link/pose_classification_basic)
|
||||
* [Pose Classification Colab (Extended)](https://mediapipe.page.link/pose_classification_extended)
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
---
|
||||
layout: default
|
||||
title: Pose Classification
|
||||
parent: Pose
|
||||
grand_parent: Solutions
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
# Pose Classification
|
||||
{: .no_toc }
|
||||
|
||||
<details close markdown="block">
|
||||
<summary>
|
||||
Table of contents
|
||||
</summary>
|
||||
{: .text-delta }
|
||||
1. TOC
|
||||
{:toc}
|
||||
</details>
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
One of the applications
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
can enable is fitness. More specifically - pose classification and repetition
|
||||
counting. In this section we'll provide basic guidance on building a custom pose
|
||||
classifier with the help of [Colabs](#colabs) and wrap it in a simple fitness
|
||||
demo within
|
||||
[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
|
||||
Push-ups and squats are used for demonstration purposes as the most common
|
||||
exercises.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 1. Pose classification and repetition counting with MediaPipe Pose.* |
|
||||
|
||||
We picked the
|
||||
[k-nearest neighbors algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm)
|
||||
(k-NN) as the classifier. It's simple and easy to start with. The algorithm
|
||||
determines the object's class based on the closest samples in the training set.
|
||||
|
||||
**To build it, one needs to:**
|
||||
|
||||
1. Collect image samples of the target exercises and run pose prediction on
|
||||
them,
|
||||
2. Convert obtained pose landmarks to a representation suitable for the k-NN
|
||||
classifier and form a training set using these [Colabs](#colabs),
|
||||
3. Perform the classification itself followed by repetition counting (e.g., in
|
||||
the
|
||||
[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app)).
|
||||
|
||||
## Training Set
|
||||
|
||||
To build a good classifier appropriate samples should be collected for the
|
||||
training set: about a few hundred samples for each terminal state of each
|
||||
exercise (e.g., "up" and "down" positions for push-ups). It's important that
|
||||
collected samples cover different camera angles, environment conditions, body
|
||||
shapes, and exercise variations.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 2. Two terminal states of push-ups.* |
|
||||
|
||||
To transform samples into a k-NN classifier training set, both
|
||||
[`Pose Classification Colab (Basic)`] and
|
||||
[`Pose Classification Colab (Extended)`] could be used. They use the
|
||||
[Python Solution API](./pose.md#python-solution-api) to run the BlazePose models
|
||||
on given images and dump predicted pose landmarks to a CSV file. Additionally,
|
||||
the [`Pose Classification Colab (Extended)`] provides useful tools to find
|
||||
outliers (e.g., wrongly predicted poses) and underrepresented classes (e.g., not
|
||||
covering all camera angles) by classifying each sample against the entire
|
||||
training set. After that, you'll be able to test the classifier on an arbitrary
|
||||
video right in the Colab.
|
||||
|
||||
## Classification
|
||||
|
||||
Code of the classifier is available both in the
|
||||
[`Pose Classification Colab (Extended)`] and in the
|
||||
[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
|
||||
Please refer to them for details of the approach described below.
|
||||
|
||||
The k-NN algorithm used for pose classification requires a feature vector
|
||||
representation of each sample and a metric to compute the distance between two
|
||||
such vectors to find the nearest pose samples to a target one.
|
||||
|
||||
To convert pose landmarks to a feature vector, we use pairwise distances between
|
||||
predefined lists of pose joints, such as distances between wrist and shoulder,
|
||||
ankle and hip, and two wrists. Since the algorithm relies on distances, all
|
||||
poses are normalized to have the same torso size and vertical torso orientation
|
||||
before the conversion.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 3. Main pairwise distances used for the pose feature vector.* |
|
||||
|
||||
To get a better classification result, k-NN search is invoked twice with
|
||||
different distance metrics:
|
||||
|
||||
* First, to filter out samples that are almost the same as the target one but
|
||||
have only a few different values in the feature vector (which means
|
||||
differently bent joints and thus other pose class), minimum per-coordinate
|
||||
distance is used as distance metric,
|
||||
* Then average per-coordinate distance is used to find the nearest pose
|
||||
cluster among those from the first search.
|
||||
|
||||
Finally, we apply
|
||||
[exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average)
|
||||
(EMA) smoothing to level any noise from pose prediction or classification. To do
|
||||
that, we search not only for the nearest pose cluster, but we calculate a
|
||||
probability for each of them and use it for smoothing over time.
|
||||
|
||||
## Repetition Counting
|
||||
|
||||
To count the repetitions, the algorithm monitors the probability of a target
|
||||
pose class. Let's take push-ups with its "up" and "down" terminal states:
|
||||
|
||||
* When the probability of the "down" pose class passes a certain threshold for
|
||||
the first time, the algorithm marks that the "down" pose class is entered.
|
||||
* Once the probability drops below the threshold, the algorithm marks that the
|
||||
"down" pose class has been exited and increases the counter.
|
||||
|
||||
To avoid cases when the probability fluctuates around the threshold (e.g., when
|
||||
the user pauses between "up" and "down" states) causing phantom counts, the
|
||||
threshold used to detect when the state is exited is actually slightly lower
|
||||
than the one used to detect when the state is entered. It creates an interval
|
||||
where the pose class and the counter can't be changed.
|
||||
|
||||
## Future Work
|
||||
|
||||
We are actively working on improving
|
||||
[BlazePose GHUM 3D](./pose.md#pose-landmark-model-blazepose-ghum-3d)'s Z
|
||||
prediction. It will allow us to use joint angles in the feature vectors, which
|
||||
are more natural and easier to configure (although distances can still be useful
|
||||
to detect touches between body parts) and to perform rotation normalization of
|
||||
poses and reduce the number of camera angles required for accurate k-NN
|
||||
classification.
|
||||
|
||||
## Colabs
|
||||
|
||||
* [`Pose Classification Colab (Basic)`]
|
||||
* [`Pose Classification Colab (Extended)`]
|
||||
|
||||
[`Pose Classification Colab (Basic)`]: https://mediapipe.page.link/pose_classification_basic
|
||||
[`Pose Classification Colab (Extended)`]: https://mediapipe.page.link/pose_classification_extended
|
||||
@@ -28,7 +28,7 @@ has_toc: false
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
|
||||
@@ -41,6 +41,7 @@ profiler_config {
|
||||
trace_enabled: true
|
||||
enable_profiler: true
|
||||
trace_log_interval_count: 200
|
||||
trace_log_path: "/sdcard/Download/"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -64,7 +65,7 @@ MediaPipe will emit data into a pre-specified directory:
|
||||
|
||||
* On the desktop, this will be the `/tmp` directory.
|
||||
|
||||
* On Android, this will be the `/sdcard` directory.
|
||||
* On Android, this will be the external storage directory (e.g., `/storage/emulated/0/`).
|
||||
|
||||
* On iOS, this can be reached through XCode. Select "Window/Devices and
|
||||
Simulators" and select the "Devices" tab.
|
||||
@@ -103,7 +104,7 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
* Include the line below in your `AndroidManifest.xml` file.
|
||||
|
||||
```xml
|
||||
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
|
||||
<uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
|
||||
```
|
||||
|
||||
* Grant the permission either upon first app launch, or by going into
|
||||
@@ -130,8 +131,8 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
events to a trace log files at:
|
||||
|
||||
```bash
|
||||
/sdcard/mediapipe_trace_0.binarypb
|
||||
/sdcard/mediapipe_trace_1.binarypb
|
||||
/storage/emulated/0/Download/mediapipe_trace_0.binarypb
|
||||
/storage/emulated/0/Download/mediapipe_trace_1.binarypb
|
||||
```
|
||||
|
||||
After every 5 sec, writing shifts to a successive trace log file, such that
|
||||
@@ -139,10 +140,10 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
trace files have been written to the device using adb shell.
|
||||
|
||||
```bash
|
||||
adb shell "ls -la /sdcard/"
|
||||
adb shell "ls -la /storage/emulated/0/Download"
|
||||
```
|
||||
|
||||
On android, MediaPipe selects the external storage directory `/sdcard` for
|
||||
On android, MediaPipe selects the external storage (e.g., `/storage/emulated/0/`) for
|
||||
trace logs. This directory can be overridden using the setting
|
||||
`trace_log_path`, like:
|
||||
|
||||
@@ -150,7 +151,7 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
profiler_config {
|
||||
trace_enabled: true
|
||||
enable_profiler: true
|
||||
trace_log_path: "/sdcard/profiles/"
|
||||
trace_log_path: "/sdcard/Download/profiles/"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -161,7 +162,7 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
|
||||
```bash
|
||||
# from your terminal
|
||||
adb pull /sdcard/mediapipe_trace_0.binarypb
|
||||
adb pull /storage/emulated/0/Download/mediapipe_trace_0.binarypb
|
||||
# if successful you should see something like
|
||||
# /sdcard/mediapipe_trace_0.binarypb: 1 file pulled. 0.1 MB/s (6766 bytes in 0.045s)
|
||||
```
|
||||
|
||||
@@ -128,7 +128,7 @@ cc_library(
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -147,7 +147,7 @@ cc_library(
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_audio_tools//audio/dsp/mfcc",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -168,7 +168,7 @@ cc_library(
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_audio_tools//audio/dsp:resampler",
|
||||
"@com_google_audio_tools//audio/dsp:resampler_q",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -208,7 +208,7 @@ cc_library(
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@com_google_audio_tools//audio/dsp/spectrogram",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -228,7 +228,7 @@ cc_library(
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -242,9 +242,9 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -261,7 +261,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -276,7 +276,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -296,7 +296,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@com_google_audio_tools//audio/dsp:number_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -314,7 +314,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -333,7 +333,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -352,6 +352,6 @@ cc_test(
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@com_google_audio_tools//audio/dsp:signal_vector_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -12,10 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
@@ -25,7 +25,7 @@ namespace mediapipe {
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, TestWAV) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
@@ -34,7 +34,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -56,7 +56,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, Test48KWAV) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
@@ -65,7 +65,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -87,7 +87,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, TestMP3) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
@@ -96,7 +96,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -118,7 +118,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, TestAAC) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
@@ -127,7 +127,7 @@ TEST(AudioDecoderCalculatorTest, TestAAC) {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
|
||||
@@ -414,7 +414,7 @@ cc_library(
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -430,7 +430,7 @@ cc_library(
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -450,6 +450,20 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "nonzero_calculator",
|
||||
srcs = ["nonzero_calculator.cc"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "mux_calculator_test",
|
||||
srcs = ["mux_calculator_test.cc"],
|
||||
@@ -776,7 +790,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -793,7 +807,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1024,7 +1038,7 @@ cc_library(
|
||||
"//mediapipe/framework/tool:status_util",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
@@ -70,7 +70,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
num_threads: 4
|
||||
input_stream: "ints"
|
||||
node {
|
||||
@@ -90,7 +90,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
|
||||
input_stream: "BATCH_END:timestamp"
|
||||
output_stream: "ITERABLE:ints_plus_one"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("ints_plus_one", &graph_config, &output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph_.StartRun({}));
|
||||
@@ -197,7 +197,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
|
||||
protected:
|
||||
void SetUp() override {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
num_threads: 4
|
||||
input_stream: "ints"
|
||||
input_stream: "force_ints_to_be_timestamp_bound_update"
|
||||
@@ -229,7 +229,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
|
||||
input_stream: "ints_plus_one"
|
||||
output_stream: "ints_plus_one_passed_through"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("ints_plus_one_passed_through", &graph_config,
|
||||
&output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config));
|
||||
@@ -338,7 +338,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
num_threads: 4
|
||||
input_stream: "ints"
|
||||
input_stream: "multiplier"
|
||||
@@ -362,7 +362,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
|
||||
input_stream: "BATCH_END:timestamp"
|
||||
output_stream: "ITERABLE:multiplied_ints"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("multiplied_ints", &graph_config, &output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph_.StartRun({}));
|
||||
|
||||
@@ -38,14 +38,14 @@ void AddInputVector(const std::vector<int>& input, int64 timestamp,
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
|
||||
std::vector<int> input = {};
|
||||
@@ -60,14 +60,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
|
||||
std::vector<int> input = {0, 1, 2, 3};
|
||||
@@ -85,14 +85,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
|
||||
{
|
||||
@@ -133,7 +133,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "TestClipUniqueIntPtrVectorSizeCalculator"
|
||||
@@ -143,7 +143,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("output_vector", &graph_config, &outputs);
|
||||
@@ -178,7 +178,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
input_side_packet: "max_vec_size"
|
||||
@@ -186,7 +186,7 @@ TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
// This should override the default of 1 set in the options.
|
||||
runner.MutableSidePackets()->Index(0) = Adopt(new int(2));
|
||||
|
||||
@@ -392,7 +392,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
input_stream: "in_3"
|
||||
@@ -403,7 +403,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
|
||||
input_stream: "in_3"
|
||||
output_stream: "out"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("out", &graph_config, &outputs);
|
||||
@@ -456,7 +456,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
node {
|
||||
@@ -465,7 +465,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
|
||||
input_stream: "in_2"
|
||||
output_stream: "out"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("out", &graph_config, &outputs);
|
||||
@@ -505,7 +505,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
node {
|
||||
@@ -519,7 +519,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("out", &graph_config, &outputs);
|
||||
|
||||
@@ -62,7 +62,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:0:int_packet"
|
||||
@@ -82,7 +82,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
@@ -111,7 +111,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:0:int_packet"
|
||||
@@ -131,7 +131,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
@@ -152,7 +152,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:int_packet"
|
||||
@@ -163,14 +163,14 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
EXPECT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:0:int_packet"
|
||||
@@ -181,7 +181,7 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
EXPECT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ namespace mediapipe {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
max_quantized_value: 2
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
@@ -51,7 +51,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
@@ -61,7 +61,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
min_quantized_value: 2
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
@@ -76,7 +76,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
@@ -86,7 +86,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
@@ -101,7 +101,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
|
||||
TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
@@ -111,7 +111,7 @@ TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
|
||||
min_quantized_value: -2
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
|
||||
@@ -57,7 +57,7 @@ namespace mediapipe {
|
||||
//
|
||||
// The "ALLOW" stream indicates the transition between accepting frames and
|
||||
// dropping frames. "ALLOW = true" indicates the start of accepting frames
|
||||
// including the current timestamp, and "ALLOW = true" indicates the start of
|
||||
// including the current timestamp, and "ALLOW = false" indicates the start of
|
||||
// dropping frames including the current timestamp.
|
||||
//
|
||||
// FlowLimiterCalculator provides limited support for multiple input streams.
|
||||
|
||||
@@ -126,7 +126,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
|
||||
// Back-edge "finished" limits processing to one frame in-flight.
|
||||
// The LambdaCalculator is used to keep certain frames in flight.
|
||||
CalculatorGraphConfig InflightGraphConfig() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
node {
|
||||
calculator: 'FlowLimiterCalculator'
|
||||
@@ -143,7 +143,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
|
||||
input_stream: 'in_1_sampled'
|
||||
output_stream: 'out_1'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
protected:
|
||||
@@ -271,7 +271,7 @@ REGISTER_CALCULATOR(DropCalculator);
|
||||
class FlowLimiterCalculatorTest : public testing::Test {
|
||||
protected:
|
||||
CalculatorGraphConfig InflightGraphConfig() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
node {
|
||||
calculator: 'FlowLimiterCalculator'
|
||||
@@ -296,7 +296,7 @@ class FlowLimiterCalculatorTest : public testing::Test {
|
||||
input_stream: 'PACKET:out_1_sampled'
|
||||
output_stream: 'PACKET:out_1'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
// Parse an absl::Time from RFC3339 format.
|
||||
@@ -348,10 +348,10 @@ TEST_F(FlowLimiterCalculatorTest, FinishedTimestamps) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config = InflightGraphConfig();
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 1
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
@@ -419,11 +419,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedLost) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config = InflightGraphConfig();
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 1
|
||||
in_flight_timeout: 100000 # 100 ms
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
@@ -483,11 +483,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedDelayed) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config = InflightGraphConfig();
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 1
|
||||
in_flight_timeout: 100000 # 100 ms
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
@@ -548,7 +548,7 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
input_stream: 'in_2'
|
||||
node {
|
||||
@@ -576,13 +576,13 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
|
||||
input_stream: 'PACKET:out_1_sampled'
|
||||
output_stream: 'PACKET:out_1'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 1
|
||||
in_flight_timeout: 100000 # 100 ms
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
@@ -657,7 +657,7 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
input_stream: 'in_2'
|
||||
node {
|
||||
@@ -685,13 +685,13 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
|
||||
input_stream: 'PACKET:out_1_sampled'
|
||||
output_stream: 'PACKET:out_1'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 0
|
||||
in_flight_timeout: 100000 # 100 ms
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
|
||||
@@ -64,13 +64,13 @@ const char kMatrixText2[] =
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "input_matrix"
|
||||
input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
input_side_packet: "MINUEND:side_matrix2"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
auto status = runner.Run();
|
||||
EXPECT_THAT(
|
||||
@@ -81,12 +81,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig) {
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
input_stream: "SUBTRAHEND:side_matrix2"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
auto status = runner.Run();
|
||||
EXPECT_THAT(status.message(), testing::HasSubstr("must be connected"));
|
||||
@@ -95,12 +95,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "MINUEND:input_matrix"
|
||||
input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
Matrix* side_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText, side_matrix);
|
||||
@@ -124,12 +124,12 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "SUBTRAHEND:input_matrix"
|
||||
input_side_packet: "MINUEND:side_matrix"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
Matrix* side_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText, side_matrix);
|
||||
|
||||
@@ -26,31 +26,33 @@ namespace {
|
||||
|
||||
// Checks that the calculator fails if no input streams are provided.
|
||||
TEST(InvariantMergeInputStreamsCalculator, NoInputStreamsMustFail) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
output_stream: "merged_output"
|
||||
)"));
|
||||
)pb"));
|
||||
// Expect calculator to fail.
|
||||
ASSERT_FALSE(runner.Run().ok());
|
||||
}
|
||||
|
||||
// Checks that the calculator fails with an incorrect number of output streams.
|
||||
TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
|
||||
CalculatorRunner runner1(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
)"));
|
||||
CalculatorRunner runner1(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
)pb"));
|
||||
// Expect calculator to fail.
|
||||
EXPECT_FALSE(runner1.Run().ok());
|
||||
|
||||
CalculatorRunner runner2(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
output_stream: "output1"
|
||||
output_stream: "output2"
|
||||
)"));
|
||||
CalculatorRunner runner2(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
output_stream: "output1"
|
||||
output_stream: "output2"
|
||||
)pb"));
|
||||
// Expect calculator to fail.
|
||||
ASSERT_FALSE(runner2.Run().ok());
|
||||
}
|
||||
@@ -58,12 +60,12 @@ TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
|
||||
// Ensures two streams with differing types can be merged correctly.
|
||||
TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
|
||||
TestMergingTwoStreams) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
output_stream: "combined_output"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// input1: integers 10, 20, 30, occurring at times 10, 20, 30.
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
@@ -102,13 +104,13 @@ TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
|
||||
// Ensures three streams with differing types can be merged correctly.
|
||||
TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
|
||||
TestMergingThreeStreams) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
input_stream: "input3"
|
||||
output_stream: "combined_output"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// input1: integer 30 occurring at time 30.
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
|
||||
@@ -31,7 +31,7 @@ namespace {
|
||||
|
||||
// Graph with default input stream handler, and the input selection is driven
|
||||
// by an input stream. All MuxCalculator inputs are present at each timestamp.
|
||||
constexpr char kTestGraphConfig1[] = R"proto(
|
||||
constexpr char kTestGraphConfig1[] = R"pb(
|
||||
input_stream: "input"
|
||||
output_stream: "test_output"
|
||||
node {
|
||||
@@ -60,12 +60,12 @@ constexpr char kTestGraphConfig1[] = R"proto(
|
||||
output_stream: "OUTPUT:test_output"
|
||||
input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
|
||||
}
|
||||
)proto";
|
||||
)pb";
|
||||
|
||||
// Graph with default input stream handler, and the input selection is driven
|
||||
// by an input side packet. All MuxCalculator inputs are present at each
|
||||
// timestamp.
|
||||
constexpr char kTestGraphConfig2[] = R"proto(
|
||||
constexpr char kTestGraphConfig2[] = R"pb(
|
||||
input_side_packet: "input_selector"
|
||||
input_stream: "input"
|
||||
output_stream: "test_output"
|
||||
@@ -93,12 +93,12 @@ constexpr char kTestGraphConfig2[] = R"proto(
|
||||
output_stream: "OUTPUT:test_output"
|
||||
input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
|
||||
}
|
||||
)proto";
|
||||
)pb";
|
||||
|
||||
// Graph with mux input stream handler, and the input selection is driven
|
||||
// by an input stream. Only one MuxCalculator input is present at each
|
||||
// timestamp.
|
||||
constexpr char kTestGraphConfig3[] = R"proto(
|
||||
constexpr char kTestGraphConfig3[] = R"pb(
|
||||
input_stream: "input"
|
||||
output_stream: "test_output"
|
||||
node {
|
||||
@@ -117,7 +117,7 @@ constexpr char kTestGraphConfig3[] = R"proto(
|
||||
input_stream: "SELECT:input_select"
|
||||
output_stream: "OUTPUT:test_output"
|
||||
}
|
||||
)proto";
|
||||
)pb";
|
||||
|
||||
constexpr char kOutputName[] = "test_output";
|
||||
constexpr char kInputName[] = "input";
|
||||
@@ -235,7 +235,7 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
|
||||
EXPECT_EQ(output, input_packets);
|
||||
}
|
||||
|
||||
constexpr char kDualInputGraphConfig[] = R"proto(
|
||||
constexpr char kDualInputGraphConfig[] = R"pb(
|
||||
input_stream: "input_0"
|
||||
input_stream: "input_1"
|
||||
input_stream: "input_select"
|
||||
@@ -247,7 +247,7 @@ constexpr char kDualInputGraphConfig[] = R"proto(
|
||||
input_stream: "SELECT:input_select"
|
||||
output_stream: "OUTPUT:test_output"
|
||||
}
|
||||
)proto";
|
||||
)pb";
|
||||
|
||||
TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
|
||||
CalculatorGraphConfig config =
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// A Calculator that returns 0 if INPUT is 0, and 1 otherwise.
|
||||
class NonZeroCalculator : public Node {
|
||||
public:
|
||||
static constexpr Input<int>::SideFallback kIn{"INPUT"};
|
||||
static constexpr Output<int> kOut{"OUTPUT"};
|
||||
|
||||
MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
if (!kIn(cc).IsEmpty()) {
|
||||
auto output = std::make_unique<int>((*kIn(cc) != 0) ? 1 : 0);
|
||||
kOut(cc).Send(std::move(output));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
MEDIAPIPE_REGISTER_NODE(NonZeroCalculator);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
@@ -40,7 +40,7 @@ MATCHER_P2(BoolPacket, value, timestamp, "") {
|
||||
TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
std::vector<Packet> output_packets;
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'allow'
|
||||
input_stream: 'value'
|
||||
node {
|
||||
@@ -54,7 +54,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
input_stream: 'PACKET:gated_value'
|
||||
output_stream: 'PRESENCE:presence'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("presence", &graph_config, &output_packets);
|
||||
|
||||
CalculatorGraph graph;
|
||||
|
||||
@@ -87,7 +87,6 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
|
||||
|
||||
flush_last_packet_ = resampler_options.flush_last_packet();
|
||||
jitter_ = resampler_options.jitter();
|
||||
jitter_with_reflection_ = resampler_options.jitter_with_reflection();
|
||||
|
||||
input_data_id_ = cc->Inputs().GetId("DATA", 0);
|
||||
if (!input_data_id_.IsValid()) {
|
||||
@@ -98,11 +97,7 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
|
||||
output_data_id_ = cc->Outputs().GetId("", 0);
|
||||
}
|
||||
|
||||
period_count_ = 0;
|
||||
frame_rate_ = resampler_options.frame_rate();
|
||||
base_timestamp_ = resampler_options.has_base_timestamp()
|
||||
? Timestamp(resampler_options.base_timestamp())
|
||||
: Timestamp::Unset();
|
||||
start_time_ = resampler_options.has_start_time()
|
||||
? Timestamp(resampler_options.start_time())
|
||||
: Timestamp::Min();
|
||||
@@ -141,30 +136,9 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
|
||||
}
|
||||
}
|
||||
|
||||
if (jitter_ != 0.0) {
|
||||
if (resampler_options.output_header() !=
|
||||
PacketResamplerCalculatorOptions::NONE) {
|
||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||
"the actual value.";
|
||||
}
|
||||
if (flush_last_packet_) {
|
||||
flush_last_packet_ = false;
|
||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::Status(
|
||||
absl::StatusCode::kInvalidArgument,
|
||||
"SecureRandom is not available. With \"jitter\" specified, "
|
||||
"PacketResamplerCalculator processing cannot proceed.");
|
||||
}
|
||||
packet_reservoir_random_ = CreateSecureRandom(seed);
|
||||
}
|
||||
packet_reservoir_ =
|
||||
std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
|
||||
return absl::OkStatus();
|
||||
strategy_ = GetSamplingStrategy(resampler_options);
|
||||
|
||||
return strategy_->Open(cc);
|
||||
}
|
||||
|
||||
absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
|
||||
@@ -177,171 +151,13 @@ absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
}
|
||||
if (jitter_ != 0.0 && random_ != nullptr) {
|
||||
// Packet reservior is used to make sure there's an output for every period,
|
||||
// e.g. partial period at the end of the stream.
|
||||
if (packet_reservoir_->IsEnabled() &&
|
||||
(first_timestamp_ == Timestamp::Unset() ||
|
||||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
|
||||
auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
|
||||
packet_reservoir_->AddSample(curr_packet);
|
||||
}
|
||||
MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
|
||||
} else {
|
||||
MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
|
||||
|
||||
if (absl::Status status = strategy_->Process(cc); !status.ok()) {
|
||||
return status; // Avoid MP_RETURN_IF_ERROR macro for external release.
|
||||
}
|
||||
|
||||
last_packet_ = cc->Inputs().Get(input_data_id_).Value();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
|
||||
next_output_timestamp_min_ = first_timestamp_;
|
||||
if (jitter_with_reflection_) {
|
||||
next_output_timestamp_ =
|
||||
first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
|
||||
return;
|
||||
}
|
||||
next_output_timestamp_ =
|
||||
first_timestamp_ + frame_time_usec_ * random_->RandFloat();
|
||||
}
|
||||
|
||||
void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
|
||||
packet_reservoir_->Clear();
|
||||
if (jitter_with_reflection_) {
|
||||
next_output_timestamp_min_ += frame_time_usec_;
|
||||
Timestamp next_output_timestamp_max_ =
|
||||
next_output_timestamp_min_ + frame_time_usec_;
|
||||
|
||||
next_output_timestamp_ += frame_time_usec_ +
|
||||
random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
|
||||
jitter_usec_;
|
||||
next_output_timestamp_ = Timestamp(ReflectBetween(
|
||||
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
||||
next_output_timestamp_max_.Value()));
|
||||
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
|
||||
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
|
||||
return;
|
||||
}
|
||||
packet_reservoir_->Disable();
|
||||
next_output_timestamp_ +=
|
||||
frame_time_usec_ *
|
||||
((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
|
||||
}
|
||||
|
||||
absl::Status PacketResamplerCalculator::ProcessWithJitter(
|
||||
CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
RET_CHECK_NE(jitter_, 0.0);
|
||||
|
||||
if (first_timestamp_ == Timestamp::Unset()) {
|
||||
first_timestamp_ = cc->InputTimestamp();
|
||||
InitializeNextOutputTimestampWithJitter();
|
||||
if (first_timestamp_ == next_output_timestamp_) {
|
||||
OutputWithinLimits(
|
||||
cc,
|
||||
cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
if (frame_time_usec_ <
|
||||
(cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
|
||||
LOG_FIRST_N(WARNING, 2)
|
||||
<< "Adding jitter is not very useful when upsampling.";
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const int64 last_diff =
|
||||
(next_output_timestamp_ - last_packet_.Timestamp()).Value();
|
||||
RET_CHECK_GT(last_diff, 0);
|
||||
const int64 curr_diff =
|
||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||
if (curr_diff > 0) {
|
||||
break;
|
||||
}
|
||||
OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
|
||||
? last_packet_
|
||||
: cc->Inputs().Get(input_data_id_).Value())
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
// From now on every time a packet is emitted the timestamp of the next
|
||||
// packet becomes known; that timestamp is stored in next_output_timestamp_.
|
||||
// The only exception to this rule is the packet emitted from Close() which
|
||||
// can only happen when jitter_with_reflection is enabled but in this case
|
||||
// next_output_timestamp_min_ is a non-decreasing lower bound of any
|
||||
// subsequent packet.
|
||||
const Timestamp timestamp_bound = jitter_with_reflection_
|
||||
? next_output_timestamp_min_
|
||||
: next_output_timestamp_;
|
||||
cc->Outputs().Get(output_data_id_).SetNextTimestampBound(timestamp_bound);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status PacketResamplerCalculator::ProcessWithoutJitter(
|
||||
CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
RET_CHECK_EQ(jitter_, 0.0);
|
||||
|
||||
if (first_timestamp_ == Timestamp::Unset()) {
|
||||
// This is the first packet, initialize the first_timestamp_.
|
||||
if (base_timestamp_ == Timestamp::Unset()) {
|
||||
// Initialize first_timestamp_ with exactly the first packet timestamp.
|
||||
first_timestamp_ = cc->InputTimestamp();
|
||||
} else {
|
||||
// Initialize first_timestamp_ with the first packet timestamp
|
||||
// aligned to the base_timestamp_.
|
||||
int64 first_index = MathUtil::SafeRound<int64, double>(
|
||||
(cc->InputTimestamp() - base_timestamp_).Seconds() * frame_rate_);
|
||||
first_timestamp_ =
|
||||
base_timestamp_ + TimestampDiffFromSeconds(first_index / frame_rate_);
|
||||
}
|
||||
if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
|
||||
cc->Outputs()
|
||||
.Tag("VIDEO_HEADER")
|
||||
.Add(new VideoHeader(video_header_), Timestamp::PreStream());
|
||||
}
|
||||
}
|
||||
const Timestamp received_timestamp = cc->InputTimestamp();
|
||||
const int64 received_timestamp_idx =
|
||||
TimestampToPeriodIndex(received_timestamp);
|
||||
// Only consider the received packet if it belongs to the current period
|
||||
// (== period_count_) or to a newer one (> period_count_).
|
||||
if (received_timestamp_idx >= period_count_) {
|
||||
// Fill the empty periods until we are in the same index as the received
|
||||
// packet.
|
||||
while (received_timestamp_idx > period_count_) {
|
||||
OutputWithinLimits(
|
||||
cc, last_packet_.At(PeriodIndexToTimestamp(period_count_)));
|
||||
++period_count_;
|
||||
}
|
||||
// Now, if the received packet has a timestamp larger than the middle of
|
||||
// the current period, we can send a packet without waiting. We send the
|
||||
// one closer to the middle.
|
||||
Timestamp target_timestamp = PeriodIndexToTimestamp(period_count_);
|
||||
if (received_timestamp >= target_timestamp) {
|
||||
bool have_last_packet = (last_packet_.Timestamp() != Timestamp::Unset());
|
||||
bool send_current =
|
||||
!have_last_packet || (received_timestamp - target_timestamp <=
|
||||
target_timestamp - last_packet_.Timestamp());
|
||||
if (send_current) {
|
||||
OutputWithinLimits(
|
||||
cc, cc->Inputs().Get(input_data_id_).Value().At(target_timestamp));
|
||||
} else {
|
||||
OutputWithinLimits(cc, last_packet_.At(target_timestamp));
|
||||
}
|
||||
++period_count_;
|
||||
}
|
||||
// TODO: Add a mechanism to the framework to allow these packets
|
||||
// to be output earlier (without waiting for a much later packet to
|
||||
// arrive)
|
||||
|
||||
// Update the bound for the next packet.
|
||||
cc->Outputs()
|
||||
.Get(output_data_id_)
|
||||
.SetNextTimestampBound(PeriodIndexToTimestamp(period_count_));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -349,17 +165,34 @@ absl::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
|
||||
if (!cc->GraphStatus().ok()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
// Emit the last packet received if we have at least one packet, but
|
||||
// haven't sent anything for its period.
|
||||
if (first_timestamp_ != Timestamp::Unset() && flush_last_packet_ &&
|
||||
TimestampToPeriodIndex(last_packet_.Timestamp()) == period_count_) {
|
||||
OutputWithinLimits(cc,
|
||||
last_packet_.At(PeriodIndexToTimestamp(period_count_)));
|
||||
|
||||
return strategy_->Close(cc);
|
||||
}
|
||||
|
||||
std::unique_ptr<PacketResamplerStrategy>
|
||||
PacketResamplerCalculator::GetSamplingStrategy(
|
||||
const PacketResamplerCalculatorOptions& options) {
|
||||
if (options.reproducible_sampling()) {
|
||||
if (!options.jitter_with_reflection()) {
|
||||
LOG(WARNING)
|
||||
<< "reproducible_sampling enabled w/ jitter_with_reflection "
|
||||
"disabled. "
|
||||
<< "reproducible_sampling always uses jitter with reflection, "
|
||||
<< "Ignoring jitter_with_reflection setting.";
|
||||
}
|
||||
return absl::make_unique<ReproducibleJitterWithReflectionStrategy>(this);
|
||||
}
|
||||
if (!packet_reservoir_->IsEmpty()) {
|
||||
OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
||||
|
||||
if (options.jitter() == 0) {
|
||||
return absl::make_unique<NoJitterStrategy>(this);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
|
||||
if (options.jitter_with_reflection()) {
|
||||
return absl::make_unique<LegacyJitterWithReflectionStrategy>(this);
|
||||
}
|
||||
|
||||
// With jitter and no reflection.
|
||||
return absl::make_unique<JitterWithoutReflectionStrategy>(this);
|
||||
}
|
||||
|
||||
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const {
|
||||
@@ -385,4 +218,479 @@ void PacketResamplerCalculator::OutputWithinLimits(CalculatorContext* cc,
|
||||
}
|
||||
}
|
||||
|
||||
absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
|
||||
const auto resampler_options =
|
||||
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
|
||||
cc->InputSidePackets(), "OPTIONS");
|
||||
|
||||
if (resampler_options.output_header() !=
|
||||
PacketResamplerCalculatorOptions::NONE) {
|
||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||
"the actual value.";
|
||||
}
|
||||
|
||||
if (calculator_->flush_last_packet_) {
|
||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::InvalidArgumentError(
|
||||
"SecureRandom is not available. With \"jitter\" specified, "
|
||||
"PacketResamplerCalculator processing cannot proceed.");
|
||||
}
|
||||
|
||||
packet_reservoir_random_ = CreateSecureRandom(seed);
|
||||
packet_reservoir_ =
|
||||
std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
|
||||
if (!packet_reservoir_->IsEmpty()) {
|
||||
LOG(INFO) << "Emitting pack from reservoir.";
|
||||
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status LegacyJitterWithReflectionStrategy::Process(
|
||||
CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
|
||||
if (packet_reservoir_->IsEnabled() &&
|
||||
(first_timestamp_ == Timestamp::Unset() ||
|
||||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
|
||||
auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
|
||||
packet_reservoir_->AddSample(curr_packet);
|
||||
}
|
||||
|
||||
if (first_timestamp_ == Timestamp::Unset()) {
|
||||
first_timestamp_ = cc->InputTimestamp();
|
||||
InitializeNextOutputTimestampWithJitter();
|
||||
if (first_timestamp_ == next_output_timestamp_) {
|
||||
calculator_->OutputWithinLimits(cc, cc->Inputs()
|
||||
.Get(calculator_->input_data_id_)
|
||||
.Value()
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
if (calculator_->frame_time_usec_ <
|
||||
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
||||
LOG_FIRST_N(WARNING, 2)
|
||||
<< "Adding jitter is not very useful when upsampling.";
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const int64 last_diff =
|
||||
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
|
||||
.Value();
|
||||
RET_CHECK_GT(last_diff, 0);
|
||||
const int64 curr_diff =
|
||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||
if (curr_diff > 0) {
|
||||
break;
|
||||
}
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, (std::abs(curr_diff) > last_diff
|
||||
? calculator_->last_packet_
|
||||
: cc->Inputs().Get(calculator_->input_data_id_).Value())
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
// From now on every time a packet is emitted the timestamp of the next
|
||||
// packet becomes known; that timestamp is stored in next_output_timestamp_.
|
||||
// The only exception to this rule is the packet emitted from Close() which
|
||||
// can only happen when jitter_with_reflection is enabled but in this case
|
||||
// next_output_timestamp_min_ is a non-decreasing lower bound of any
|
||||
// subsequent packet.
|
||||
const Timestamp timestamp_bound = next_output_timestamp_min_;
|
||||
cc->Outputs()
|
||||
.Get(calculator_->output_data_id_)
|
||||
.SetNextTimestampBound(timestamp_bound);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void LegacyJitterWithReflectionStrategy::
|
||||
InitializeNextOutputTimestampWithJitter() {
|
||||
next_output_timestamp_min_ = first_timestamp_;
|
||||
next_output_timestamp_ =
|
||||
first_timestamp_ +
|
||||
random_->UnbiasedUniform64(calculator_->frame_time_usec_);
|
||||
}
|
||||
|
||||
void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
|
||||
packet_reservoir_->Clear();
|
||||
next_output_timestamp_min_ += calculator_->frame_time_usec_;
|
||||
Timestamp next_output_timestamp_max_ =
|
||||
next_output_timestamp_min_ + calculator_->frame_time_usec_;
|
||||
|
||||
next_output_timestamp_ +=
|
||||
calculator_->frame_time_usec_ +
|
||||
random_->UnbiasedUniform64(2 * calculator_->jitter_usec_ + 1) -
|
||||
calculator_->jitter_usec_;
|
||||
next_output_timestamp_ = Timestamp(ReflectBetween(
|
||||
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
||||
next_output_timestamp_max_.Value()));
|
||||
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
|
||||
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
|
||||
}
|
||||
|
||||
absl::Status ReproducibleJitterWithReflectionStrategy::Open(
|
||||
CalculatorContext* cc) {
|
||||
const auto resampler_options =
|
||||
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
|
||||
cc->InputSidePackets(), "OPTIONS");
|
||||
|
||||
if (resampler_options.output_header() !=
|
||||
PacketResamplerCalculatorOptions::NONE) {
|
||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||
"the actual value.";
|
||||
}
|
||||
|
||||
if (calculator_->flush_last_packet_) {
|
||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::InvalidArgumentError(
|
||||
"SecureRandom is not available. With \"jitter\" specified, "
|
||||
"PacketResamplerCalculator processing cannot proceed.");
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status ReproducibleJitterWithReflectionStrategy::Close(
|
||||
CalculatorContext* cc) {
|
||||
// If last packet is non-empty and a packet hasn't been emitted for this
|
||||
// period, emit the last packet.
|
||||
if (!calculator_->last_packet_.IsEmpty() && !packet_emitted_this_period_) {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, calculator_->last_packet_.At(next_output_timestamp_));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status ReproducibleJitterWithReflectionStrategy::Process(
|
||||
CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
|
||||
Packet current_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
|
||||
|
||||
if (calculator_->last_packet_.IsEmpty()) {
|
||||
// last_packet is empty, this is the first packet of the stream.
|
||||
|
||||
InitializeNextOutputTimestamp(current_packet.Timestamp());
|
||||
|
||||
// If next_output_timestamp_ happens to fall before current_packet, emit
|
||||
// current packet. Only a single packet can be emitted at the beginning
|
||||
// of the stream.
|
||||
if (next_output_timestamp_ < current_packet.Timestamp()) {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, current_packet.At(next_output_timestamp_));
|
||||
packet_emitted_this_period_ = true;
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// Last packet is set, so we are mid-stream.
|
||||
if (calculator_->frame_time_usec_ <
|
||||
(current_packet.Timestamp() - calculator_->last_packet_.Timestamp())
|
||||
.Value()) {
|
||||
// Note, if the stream is upsampling, this could lead to the same packet
|
||||
// being emitted twice. Upsampling and jitter doesn't make much sense
|
||||
// but does technically work.
|
||||
LOG_FIRST_N(WARNING, 2)
|
||||
<< "Adding jitter is not very useful when upsampling.";
|
||||
}
|
||||
|
||||
// Since we may be upsampling, we need to iteratively advance the
|
||||
// next_output_timestamp_ one period at a time until it reaches the period
|
||||
// current_packet is in. During this process, last_packet and/or
|
||||
// current_packet may be repeatly emitted.
|
||||
|
||||
UpdateNextOutputTimestamp(current_packet.Timestamp());
|
||||
|
||||
while (!packet_emitted_this_period_ &&
|
||||
next_output_timestamp_ <= current_packet.Timestamp()) {
|
||||
// last_packet < next_output_timestamp_ <= current_packet,
|
||||
// so emit the closest packet.
|
||||
Packet packet_to_emit =
|
||||
current_packet.Timestamp() - next_output_timestamp_ <
|
||||
next_output_timestamp_ - calculator_->last_packet_.Timestamp()
|
||||
? current_packet
|
||||
: calculator_->last_packet_;
|
||||
calculator_->OutputWithinLimits(cc,
|
||||
packet_to_emit.At(next_output_timestamp_));
|
||||
|
||||
packet_emitted_this_period_ = true;
|
||||
|
||||
// If we are upsampling, packet_emitted_this_period_ can be reset by
|
||||
// the following UpdateNext and the loop will iterate.
|
||||
UpdateNextOutputTimestamp(current_packet.Timestamp());
|
||||
}
|
||||
|
||||
// Set the bounds on the output stream. Note, if we emitted a packet
|
||||
// above, it will already be set at next_output_timestamp_ + 1, in which
|
||||
// case we have to skip setting it.
|
||||
if (cc->Outputs().Get(calculator_->output_data_id_).NextTimestampBound() <
|
||||
next_output_timestamp_) {
|
||||
cc->Outputs()
|
||||
.Get(calculator_->output_data_id_)
|
||||
.SetNextTimestampBound(next_output_timestamp_);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void ReproducibleJitterWithReflectionStrategy::InitializeNextOutputTimestamp(
|
||||
Timestamp current_timestamp) {
|
||||
if (next_output_timestamp_min_ != Timestamp::Unset()) {
|
||||
return;
|
||||
}
|
||||
|
||||
next_output_timestamp_min_ = Timestamp(0);
|
||||
next_output_timestamp_ =
|
||||
Timestamp(GetNextRandom(calculator_->frame_time_usec_));
|
||||
|
||||
// While the current timestamp is ahead of the max (i.e. min + frame_time),
|
||||
// fast-forward.
|
||||
while (current_timestamp >=
|
||||
next_output_timestamp_min_ + calculator_->frame_time_usec_) {
|
||||
packet_emitted_this_period_ = true; // Force update...
|
||||
UpdateNextOutputTimestamp(current_timestamp);
|
||||
}
|
||||
}
|
||||
|
||||
void ReproducibleJitterWithReflectionStrategy::UpdateNextOutputTimestamp(
|
||||
Timestamp current_timestamp) {
|
||||
if (packet_emitted_this_period_ &&
|
||||
current_timestamp >=
|
||||
next_output_timestamp_min_ + calculator_->frame_time_usec_) {
|
||||
next_output_timestamp_min_ += calculator_->frame_time_usec_;
|
||||
Timestamp next_output_timestamp_max_ =
|
||||
next_output_timestamp_min_ + calculator_->frame_time_usec_;
|
||||
|
||||
next_output_timestamp_ += calculator_->frame_time_usec_ +
|
||||
GetNextRandom(2 * calculator_->jitter_usec_ + 1) -
|
||||
calculator_->jitter_usec_;
|
||||
next_output_timestamp_ = Timestamp(ReflectBetween(
|
||||
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
||||
next_output_timestamp_max_.Value()));
|
||||
|
||||
packet_emitted_this_period_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
|
||||
const auto resampler_options =
|
||||
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
|
||||
cc->InputSidePackets(), "OPTIONS");
|
||||
|
||||
if (resampler_options.output_header() !=
|
||||
PacketResamplerCalculatorOptions::NONE) {
|
||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||
"the actual value.";
|
||||
}
|
||||
|
||||
if (calculator_->flush_last_packet_) {
|
||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::InvalidArgumentError(
|
||||
"SecureRandom is not available. With \"jitter\" specified, "
|
||||
"PacketResamplerCalculator processing cannot proceed.");
|
||||
}
|
||||
|
||||
packet_reservoir_random_ = CreateSecureRandom(seed);
|
||||
packet_reservoir_ =
|
||||
absl::make_unique<PacketReservoir>(packet_reservoir_random_.get());
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status JitterWithoutReflectionStrategy::Close(CalculatorContext* cc) {
|
||||
if (!packet_reservoir_->IsEmpty()) {
|
||||
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
|
||||
// Packet reservior is used to make sure there's an output for every period,
|
||||
// e.g. partial period at the end of the stream.
|
||||
if (packet_reservoir_->IsEnabled() &&
|
||||
(calculator_->first_timestamp_ == Timestamp::Unset() ||
|
||||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
|
||||
auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
|
||||
packet_reservoir_->AddSample(curr_packet);
|
||||
}
|
||||
|
||||
if (calculator_->first_timestamp_ == Timestamp::Unset()) {
|
||||
calculator_->first_timestamp_ = cc->InputTimestamp();
|
||||
InitializeNextOutputTimestamp();
|
||||
if (calculator_->first_timestamp_ == next_output_timestamp_) {
|
||||
calculator_->OutputWithinLimits(cc, cc->Inputs()
|
||||
.Get(calculator_->input_data_id_)
|
||||
.Value()
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestamp();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
if (calculator_->frame_time_usec_ <
|
||||
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
||||
LOG_FIRST_N(WARNING, 2)
|
||||
<< "Adding jitter is not very useful when upsampling.";
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const int64 last_diff =
|
||||
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
|
||||
.Value();
|
||||
RET_CHECK_GT(last_diff, 0);
|
||||
const int64 curr_diff =
|
||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||
if (curr_diff > 0) {
|
||||
break;
|
||||
}
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, (std::abs(curr_diff) > last_diff
|
||||
? calculator_->last_packet_
|
||||
: cc->Inputs().Get(calculator_->input_data_id_).Value())
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestamp();
|
||||
cc->Outputs()
|
||||
.Get(calculator_->output_data_id_)
|
||||
.SetNextTimestampBound(next_output_timestamp_);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void JitterWithoutReflectionStrategy::InitializeNextOutputTimestamp() {
|
||||
next_output_timestamp_min_ = calculator_->first_timestamp_;
|
||||
next_output_timestamp_ = calculator_->first_timestamp_ +
|
||||
calculator_->frame_time_usec_ * random_->RandFloat();
|
||||
}
|
||||
|
||||
void JitterWithoutReflectionStrategy::UpdateNextOutputTimestamp() {
|
||||
packet_reservoir_->Clear();
|
||||
packet_reservoir_->Disable();
|
||||
next_output_timestamp_ += calculator_->frame_time_usec_ *
|
||||
((1.0 - calculator_->jitter_) +
|
||||
2.0 * calculator_->jitter_ * random_->RandFloat());
|
||||
}
|
||||
|
||||
absl::Status NoJitterStrategy::Open(CalculatorContext* cc) {
|
||||
const auto resampler_options =
|
||||
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
|
||||
cc->InputSidePackets(), "OPTIONS");
|
||||
base_timestamp_ = resampler_options.has_base_timestamp()
|
||||
? Timestamp(resampler_options.base_timestamp())
|
||||
: Timestamp::Unset();
|
||||
|
||||
period_count_ = 0;
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status NoJitterStrategy::Close(CalculatorContext* cc) {
|
||||
// Emit the last packet received if we have at least one packet, but
|
||||
// haven't sent anything for its period.
|
||||
if (calculator_->first_timestamp_ != Timestamp::Unset() &&
|
||||
calculator_->flush_last_packet_ &&
|
||||
calculator_->TimestampToPeriodIndex(
|
||||
calculator_->last_packet_.Timestamp()) == period_count_) {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, calculator_->last_packet_.At(
|
||||
calculator_->PeriodIndexToTimestamp(period_count_)));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
|
||||
if (calculator_->first_timestamp_ == Timestamp::Unset()) {
|
||||
// This is the first packet, initialize the first_timestamp_.
|
||||
if (base_timestamp_ == Timestamp::Unset()) {
|
||||
// Initialize first_timestamp_ with exactly the first packet timestamp.
|
||||
calculator_->first_timestamp_ = cc->InputTimestamp();
|
||||
} else {
|
||||
// Initialize first_timestamp_ with the first packet timestamp
|
||||
// aligned to the base_timestamp_.
|
||||
int64 first_index = MathUtil::SafeRound<int64, double>(
|
||||
(cc->InputTimestamp() - base_timestamp_).Seconds() *
|
||||
calculator_->frame_rate_);
|
||||
calculator_->first_timestamp_ =
|
||||
base_timestamp_ +
|
||||
TimestampDiffFromSeconds(first_index / calculator_->frame_rate_);
|
||||
}
|
||||
if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
|
||||
cc->Outputs()
|
||||
.Tag("VIDEO_HEADER")
|
||||
.Add(new VideoHeader(calculator_->video_header_),
|
||||
Timestamp::PreStream());
|
||||
}
|
||||
}
|
||||
const Timestamp received_timestamp = cc->InputTimestamp();
|
||||
const int64 received_timestamp_idx =
|
||||
calculator_->TimestampToPeriodIndex(received_timestamp);
|
||||
// Only consider the received packet if it belongs to the current period
|
||||
// (== period_count_) or to a newer one (> period_count_).
|
||||
if (received_timestamp_idx >= period_count_) {
|
||||
// Fill the empty periods until we are in the same index as the received
|
||||
// packet.
|
||||
while (received_timestamp_idx > period_count_) {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, calculator_->last_packet_.At(
|
||||
calculator_->PeriodIndexToTimestamp(period_count_)));
|
||||
++period_count_;
|
||||
}
|
||||
// Now, if the received packet has a timestamp larger than the middle of
|
||||
// the current period, we can send a packet without waiting. We send the
|
||||
// one closer to the middle.
|
||||
Timestamp target_timestamp =
|
||||
calculator_->PeriodIndexToTimestamp(period_count_);
|
||||
if (received_timestamp >= target_timestamp) {
|
||||
bool have_last_packet =
|
||||
(calculator_->last_packet_.Timestamp() != Timestamp::Unset());
|
||||
bool send_current =
|
||||
!have_last_packet ||
|
||||
(received_timestamp - target_timestamp <=
|
||||
target_timestamp - calculator_->last_packet_.Timestamp());
|
||||
if (send_current) {
|
||||
calculator_->OutputWithinLimits(cc,
|
||||
cc->Inputs()
|
||||
.Get(calculator_->input_data_id_)
|
||||
.Value()
|
||||
.At(target_timestamp));
|
||||
} else {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, calculator_->last_packet_.At(target_timestamp));
|
||||
}
|
||||
++period_count_;
|
||||
}
|
||||
// TODO: Add a mechanism to the framework to allow these packets
|
||||
// to be output earlier (without waiting for a much later packet to
|
||||
// arrive)
|
||||
|
||||
// Update the bound for the next packet.
|
||||
cc->Outputs()
|
||||
.Get(calculator_->output_data_id_)
|
||||
.SetNextTimestampBound(
|
||||
calculator_->PeriodIndexToTimestamp(period_count_));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -55,7 +55,7 @@ class PacketReservoir {
|
||||
// correspond to timestamp t.
|
||||
// - The next packet is chosen randomly (uniform distribution) among frames
|
||||
// that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
|
||||
// - if jitter_with_reflection_ is true, the timestamp will be reflected
|
||||
// - if jitter_with_reflection is true, the timestamp will be reflected
|
||||
// against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
|
||||
// so that its marginal distribution is uniform within this interval.
|
||||
// In the formula, t_0 is the timestamp of the first sampled
|
||||
@@ -66,6 +66,17 @@ class PacketReservoir {
|
||||
// the resampling. For Cloud ML Video Intelligence API, the hash of the
|
||||
// input video should serve this purpose. For YouTube, either video ID or
|
||||
// content hex ID of the input video should do.
|
||||
// - If reproducible_samping is true, care is taken to allow reproducible
|
||||
// "mid-stream" sampling. The calculator can be executed on a stream that
|
||||
// doesn't start at the first period. For instance, if the calculator
|
||||
// is run on a 10 second stream it will produce the same set of samples
|
||||
// as two runs of the calculator, the first with 3 seconds of input starting
|
||||
// at time 0 and the second with 7 seconds of input starting at time +3s.
|
||||
// - In order to guarantee the exact same samples, 1) the inputs must be
|
||||
// aligned with the sampling period. For instance, if the sampling rate
|
||||
// is 2 frames per second, streams should be aligned on 0.5 second
|
||||
// boundaries, and 2) the stream must include at least one extra packet
|
||||
// before and after the second aligned sampling period.
|
||||
//
|
||||
// If jitter_ is not specified:
|
||||
// - The first packet defines the first_timestamp of the output stream,
|
||||
@@ -105,19 +116,6 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Calculates the first sampled timestamp that incorporates a jittering
|
||||
// offset.
|
||||
void InitializeNextOutputTimestampWithJitter();
|
||||
// Calculates the next sampled timestamp that incorporates a jittering offset.
|
||||
void UpdateNextOutputTimestampWithJitter();
|
||||
|
||||
// Logic for Process() when jitter_ != 0.0.
|
||||
absl::Status ProcessWithJitter(CalculatorContext* cc);
|
||||
|
||||
// Logic for Process() when jitter_ == 0.0.
|
||||
absl::Status ProcessWithoutJitter(CalculatorContext* cc);
|
||||
|
||||
// Given the current count of periods that have passed, this returns
|
||||
// the next valid timestamp of the middle point of the next period:
|
||||
// if count is 0, it returns the first_timestamp_.
|
||||
@@ -141,6 +139,16 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
// Outputs a packet if it is in range (start_time_, end_time_).
|
||||
void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
|
||||
|
||||
protected:
|
||||
// Returns Sampling Strategy to use.
|
||||
//
|
||||
// Virtual to allow injection of testing strategies.
|
||||
virtual std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
|
||||
const mediapipe::PacketResamplerCalculatorOptions& options);
|
||||
|
||||
private:
|
||||
std::unique_ptr<class PacketResamplerStrategy> strategy_;
|
||||
|
||||
// The timestamp of the first packet received.
|
||||
Timestamp first_timestamp_;
|
||||
|
||||
@@ -150,14 +158,6 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
// Inverse of frame_rate_.
|
||||
int64 frame_time_usec_;
|
||||
|
||||
// Number of periods that have passed (= #packets sent to the output).
|
||||
//
|
||||
// Can only be used if jitter_ equals zero.
|
||||
int64 period_count_;
|
||||
|
||||
// The last packet that was received.
|
||||
Packet last_packet_;
|
||||
|
||||
VideoHeader video_header_;
|
||||
// The "DATA" input stream.
|
||||
CollectionItemId input_data_id_;
|
||||
@@ -165,23 +165,15 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
CollectionItemId output_data_id_;
|
||||
|
||||
// Indicator whether to flush last packet even if its timestamp is greater
|
||||
// than the final stream timestamp. Set to false when jitter_ is non-zero.
|
||||
// than the final stream timestamp.
|
||||
bool flush_last_packet_;
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
double jitter_ = 0.0;
|
||||
bool jitter_with_reflection_;
|
||||
int64 jitter_usec_;
|
||||
Timestamp next_output_timestamp_;
|
||||
// If jittering_with_reflection_ is true, next_output_timestamp_ will be
|
||||
// kept within the interval
|
||||
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
|
||||
Timestamp next_output_timestamp_min_;
|
||||
|
||||
// If specified, output timestamps are aligned with base_timestamp.
|
||||
// Otherwise, they are aligned with the first input timestamp.
|
||||
Timestamp base_timestamp_;
|
||||
int64 jitter_usec_;
|
||||
|
||||
// The last packet that was received.
|
||||
Packet last_packet_;
|
||||
|
||||
// If specified, only outputs at/after start_time are included.
|
||||
Timestamp start_time_;
|
||||
@@ -191,15 +183,210 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
|
||||
// If set, the output timestamps nearest to start_time and end_time
|
||||
// are included in the output, even if the nearest timestamp is not
|
||||
// between start_time and end_time.W
|
||||
// between start_time and end_time.
|
||||
bool round_limits_;
|
||||
|
||||
// Allow strategies access to all internal calculator state.
|
||||
//
|
||||
// The calculator and strategies are intimiately tied together so this should
|
||||
// not break encapsulation.
|
||||
friend class LegacyJitterWithReflectionStrategy;
|
||||
friend class ReproducibleJitterWithReflectionStrategy;
|
||||
friend class JitterWithoutReflectionStrategy;
|
||||
friend class NoJitterStrategy;
|
||||
};
|
||||
|
||||
// Abstract class encapsulating sampling stategy.
|
||||
//
|
||||
// These are used solely by PacketResamplerCalculator, but are exposed here
|
||||
// to facilitate tests.
|
||||
class PacketResamplerStrategy {
|
||||
public:
|
||||
PacketResamplerStrategy(PacketResamplerCalculator* calculator)
|
||||
: calculator_(calculator) {}
|
||||
virtual ~PacketResamplerStrategy() = default;
|
||||
|
||||
// Delegate for CalculatorBase::Open. See CalculatorBase for relevant
|
||||
// implementation considerations.
|
||||
virtual absl::Status Open(CalculatorContext* cc) = 0;
|
||||
// Delegate for CalculatorBase::Close. See CalculatorBase for relevant
|
||||
// implementation considerations.
|
||||
virtual absl::Status Close(CalculatorContext* cc) = 0;
|
||||
// Delegate for CalculatorBase::Process. See CalculatorBase for relevant
|
||||
// implementation considerations.
|
||||
virtual absl::Status Process(CalculatorContext* cc) = 0;
|
||||
|
||||
protected:
|
||||
// Calculator running strategy.
|
||||
PacketResamplerCalculator* calculator_;
|
||||
};
|
||||
|
||||
// Strategy that applies Jitter with reflection based sampling.
|
||||
//
|
||||
// Used by PacketResamplerCalculator when both Jitter and reflection are
|
||||
// enabled.
|
||||
//
|
||||
// This applies the legacy jitter with reflection which doesn't allow
|
||||
// for reproducibility of sampling when starting mid-stream. This is maintained
|
||||
// for backward compatibility.
|
||||
class LegacyJitterWithReflectionStrategy : public PacketResamplerStrategy {
|
||||
public:
|
||||
LegacyJitterWithReflectionStrategy(PacketResamplerCalculator* calculator)
|
||||
: PacketResamplerStrategy(calculator) {}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
void InitializeNextOutputTimestampWithJitter();
|
||||
void UpdateNextOutputTimestampWithJitter();
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
|
||||
// The timestamp of the first packet received.
|
||||
Timestamp first_timestamp_;
|
||||
|
||||
// Next packet to be emitted. Since packets may not align perfectly with
|
||||
// next_output_timestamp_, the closest packet will be emitted.
|
||||
Timestamp next_output_timestamp_;
|
||||
|
||||
// Lower bound for next timestamp.
|
||||
//
|
||||
// next_output_timestamp_ will be kept within the interval
|
||||
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
|
||||
Timestamp next_output_timestamp_min_ = Timestamp::Unset();
|
||||
|
||||
// packet reservior used for sampling random packet out of partial
|
||||
// period when jitter is enabled
|
||||
std::unique_ptr<PacketReservoir> packet_reservoir_;
|
||||
|
||||
// random number generator used in packet_reservior_.
|
||||
std::unique_ptr<RandomBase> packet_reservoir_random_;
|
||||
};
|
||||
|
||||
// Strategy that applies reproducible jitter with reflection based sampling.
|
||||
//
|
||||
// Used by PacketResamplerCalculator when both Jitter and reflection are
|
||||
// enabled.
|
||||
class ReproducibleJitterWithReflectionStrategy
|
||||
: public PacketResamplerStrategy {
|
||||
public:
|
||||
ReproducibleJitterWithReflectionStrategy(
|
||||
PacketResamplerCalculator* calculator)
|
||||
: PacketResamplerStrategy(calculator) {}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
protected:
|
||||
// Returns next random in range (0,n].
|
||||
//
|
||||
// Exposed as virtual function for testing Jitter with reflection.
|
||||
// This is the only way random_ is accessed.
|
||||
virtual uint64 GetNextRandom(uint64 n) {
|
||||
return random_->UnbiasedUniform64(n);
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes Jitter with reflection.
|
||||
//
|
||||
// This will fast-forward to the period containing current_timestamp.
|
||||
// next_output_timestamp_ is guarnateed to be current_timestamp's period
|
||||
// and packet_emitted_this_period_ will be set to false.
|
||||
void InitializeNextOutputTimestamp(Timestamp current_timestamp);
|
||||
|
||||
// Potentially advances next_output_timestamp_ a single period.
|
||||
//
|
||||
// next_output_timestamp_ will only be advanced if packet_emitted_this_period_
|
||||
// is false. next_output_timestamp_ will never be advanced beyond
|
||||
// current_timestamp's period.
|
||||
//
|
||||
// However, next_output_timestamp_ could fall before current_timestamp's
|
||||
// period since only a single period can be advanced at a time.
|
||||
void UpdateNextOutputTimestamp(Timestamp current_timestamp);
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
|
||||
// Next packet to be emitted. Since packets may not align perfectly with
|
||||
// next_output_timestamp_, the closest packet will be emitted.
|
||||
Timestamp next_output_timestamp_;
|
||||
|
||||
// Lower bound for next timestamp.
|
||||
//
|
||||
// next_output_timestamp_ will be kept within the interval
|
||||
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
|
||||
Timestamp next_output_timestamp_min_ = Timestamp::Unset();
|
||||
|
||||
// Indicates packet was emitted for current period (i.e. the period
|
||||
// next_output_timestamp_ falls in.
|
||||
bool packet_emitted_this_period_ = false;
|
||||
};
|
||||
|
||||
// Strategy that applies Jitter without reflection based sampling.
|
||||
//
|
||||
// Used by PacketResamplerCalculator when Jitter is enabled and reflection is
|
||||
// not enabled.
|
||||
class JitterWithoutReflectionStrategy : public PacketResamplerStrategy {
|
||||
public:
|
||||
JitterWithoutReflectionStrategy(PacketResamplerCalculator* calculator)
|
||||
: PacketResamplerStrategy(calculator) {}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Calculates the first sampled timestamp that incorporates a jittering
|
||||
// offset.
|
||||
void InitializeNextOutputTimestamp();
|
||||
|
||||
// Calculates the next sampled timestamp that incorporates a jittering offset.
|
||||
void UpdateNextOutputTimestamp();
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
|
||||
// Next packet to be emitted. Since packets may not align perfectly with
|
||||
// next_output_timestamp_, the closest packet will be emitted.
|
||||
Timestamp next_output_timestamp_;
|
||||
|
||||
// Lower bound for next timestamp.
|
||||
//
|
||||
// next_output_timestamp_ will be kept within the interval
|
||||
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
|
||||
Timestamp next_output_timestamp_min_ = Timestamp::Unset();
|
||||
|
||||
// packet reservior used for sampling random packet out of partial period.
|
||||
std::unique_ptr<PacketReservoir> packet_reservoir_;
|
||||
|
||||
// random number generator used in packet_reservior_.
|
||||
std::unique_ptr<RandomBase> packet_reservoir_random_;
|
||||
};
|
||||
|
||||
// Strategy that applies sampling without any jitter.
|
||||
//
|
||||
// Used by PacketResamplerCalculator when jitter is not enabled.
|
||||
class NoJitterStrategy : public PacketResamplerStrategy {
|
||||
public:
|
||||
NoJitterStrategy(PacketResamplerCalculator* calculator)
|
||||
: PacketResamplerStrategy(calculator) {}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Number of periods that have passed (= #packets sent to the output).
|
||||
int64 period_count_;
|
||||
|
||||
// If specified, output timestamps are aligned with base_timestamp.
|
||||
// Otherwise, they are aligned with the first input timestamp.
|
||||
Timestamp base_timestamp_;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
#endif // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
|
||||
|
||||
@@ -68,8 +68,23 @@ message PacketResamplerCalculatorOptions {
|
||||
// pseudo-random number generator does its job and the number of frames is
|
||||
// sufficiently large, the average frame rate will be close to this value.
|
||||
optional double jitter = 4;
|
||||
|
||||
// Enables reflection when applying jitter.
|
||||
//
|
||||
// This option is ignored when reproducible_sampling is true, in which case
|
||||
// reflection will be used.
|
||||
//
|
||||
// New use cases should use reproducible_sampling = true, as
|
||||
// jitter_with_reflection is deprecated and will be removed at some point.
|
||||
optional bool jitter_with_reflection = 9 [default = false];
|
||||
|
||||
// If set, enabled reproducible sampling, allowing frames to be sampled
|
||||
// without regards to where the stream starts. See
|
||||
// packet_resampler_calculator.h for details.
|
||||
//
|
||||
// This enables reflection (ignoring jitter_with_reflection setting).
|
||||
optional bool reproducible_sampling = 10 [default = false];
|
||||
|
||||
// If specified, output timestamps are aligned with base_timestamp.
|
||||
// Otherwise, they are aligned with the first input timestamp.
|
||||
//
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
using ::testing::ElementsAre;
|
||||
namespace {
|
||||
// A simple version of CalculatorRunner with built-in convenience
|
||||
// methods for setting inputs from a vector and checking outputs
|
||||
@@ -96,6 +97,77 @@ class SimpleRunner : public CalculatorRunner {
|
||||
static int static_count_;
|
||||
};
|
||||
|
||||
// Matcher for Packets with uint64 payload, comparing arg packet's
|
||||
// timestamp and uint64 payload.
|
||||
MATCHER_P2(PacketAtTimestamp, payload, timestamp,
|
||||
absl::StrCat(negation ? "isn't" : "is", " a packet with payload ",
|
||||
payload, " @ time ", timestamp)) {
|
||||
if (timestamp != arg.Timestamp().Value()) {
|
||||
*result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
|
||||
return false;
|
||||
}
|
||||
int64 actual_payload = arg.template Get<int64>();
|
||||
if (actual_payload != payload) {
|
||||
*result_listener << "with incorrect payload = " << actual_payload;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// JitterWithReflectionStrategy child class which injects a specified stream
|
||||
// of "random" numbers.
|
||||
//
|
||||
// Calculators are created through factory methods, making testing and injection
|
||||
// tricky. This class utilizes a static variable, random_sequence, to pass
|
||||
// the desired random sequence into the calculator.
|
||||
class ReproducibleJitterWithReflectionStrategyForTesting
|
||||
: public ReproducibleJitterWithReflectionStrategy {
|
||||
public:
|
||||
ReproducibleJitterWithReflectionStrategyForTesting(
|
||||
PacketResamplerCalculator* calculator)
|
||||
: ReproducibleJitterWithReflectionStrategy(calculator) {}
|
||||
|
||||
// Statically accessed random sequence to use for jitter with reflection.
|
||||
//
|
||||
// An EXPECT will fail if sequence is less than the number requested during
|
||||
// processing.
|
||||
static std::vector<uint64> random_sequence;
|
||||
|
||||
protected:
|
||||
virtual uint64 GetNextRandom(uint64 n) {
|
||||
EXPECT_LT(sequence_index_, random_sequence.size());
|
||||
return random_sequence[sequence_index_++] % n;
|
||||
}
|
||||
|
||||
private:
|
||||
int32 sequence_index_ = 0;
|
||||
};
|
||||
std::vector<uint64>
|
||||
ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
|
||||
|
||||
// PacketResamplerCalculator child class which injects a specified stream
|
||||
// of "random" numbers.
|
||||
//
|
||||
// Calculators are created through factory methods, making testing and injection
|
||||
// tricky. This class utilizes a static variable, random_sequence, to pass
|
||||
// the desired random sequence into the calculator.
|
||||
class ReproducibleResamplerCalculatorForTesting
|
||||
: public PacketResamplerCalculator {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
return PacketResamplerCalculator::GetContract(cc);
|
||||
}
|
||||
|
||||
protected:
|
||||
std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
|
||||
const mediapipe::PacketResamplerCalculatorOptions& Options) {
|
||||
return absl::make_unique<
|
||||
ReproducibleJitterWithReflectionStrategyForTesting>(this);
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(ReproducibleResamplerCalculatorForTesting);
|
||||
|
||||
int SimpleRunner::static_count_ = 0;
|
||||
|
||||
TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {
|
||||
@@ -380,7 +452,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
|
||||
}
|
||||
|
||||
TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "PacketResamplerCalculator"
|
||||
input_stream: "DATA:in_data"
|
||||
input_stream: "VIDEO_HEADER:in_video_header"
|
||||
@@ -389,7 +461,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
||||
options {
|
||||
[mediapipe.PacketResamplerCalculatorOptions.ext] { frame_rate: 50.0 }
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(
|
||||
@@ -633,7 +705,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
|
||||
|
||||
TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "PacketResamplerCalculator"
|
||||
input_side_packet: "OPTIONS:options"
|
||||
input_stream: "input"
|
||||
@@ -643,16 +715,16 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
frame_rate: 60
|
||||
base_timestamp: 0
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
|
||||
{
|
||||
SimpleRunner runner(node_config);
|
||||
auto options =
|
||||
new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(
|
||||
R"(
|
||||
R"pb(
|
||||
[mediapipe.PacketResamplerCalculatorOptions.ext] {
|
||||
frame_rate: 30
|
||||
})"));
|
||||
})pb"));
|
||||
runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -662,12 +734,12 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
SimpleRunner runner(node_config);
|
||||
|
||||
auto options =
|
||||
new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"(
|
||||
new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"pb(
|
||||
merge_fields: false
|
||||
[mediapipe.PacketResamplerCalculatorOptions.ext] {
|
||||
frame_rate: 30
|
||||
base_timestamp: 0
|
||||
})"));
|
||||
})pb"));
|
||||
runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
|
||||
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
|
||||
@@ -69,7 +69,7 @@ MATCHER_P2(PairPacket, timestamp, pair, "") {
|
||||
TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
std::vector<Packet> in_prev;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -93,7 +93,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
input_stream: 'previous2'
|
||||
output_stream: 'pair'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("pair", &graph_config_, &in_prev);
|
||||
|
||||
CalculatorGraph graph_;
|
||||
@@ -169,7 +169,7 @@ REGISTER_CALCULATOR(PacketOnCloseCalculator);
|
||||
TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
|
||||
std::vector<Packet> outputs;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -192,7 +192,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
|
||||
input_stream: 'out'
|
||||
output_stream: 'close_out'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("close_out", &graph_config_, &outputs);
|
||||
|
||||
CalculatorGraph graph_;
|
||||
@@ -231,7 +231,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
|
||||
TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
|
||||
std::vector<Packet> out_and_previous_packets;
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -253,7 +253,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
|
||||
input_stream: 'previous'
|
||||
output_stream: 'out_and_previous'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("out_and_previous", &graph_config,
|
||||
&out_and_previous_packets);
|
||||
|
||||
@@ -278,7 +278,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
|
||||
TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
|
||||
std::vector<Packet> out_and_previous_packets;
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -300,7 +300,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
|
||||
input_stream: 'previous'
|
||||
output_stream: 'out_and_previous'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("out_and_previous", &graph_config,
|
||||
&out_and_previous_packets);
|
||||
|
||||
@@ -331,7 +331,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
|
||||
TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
|
||||
std::vector<Packet> outputs;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -354,7 +354,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
|
||||
input_stream: 'out'
|
||||
output_stream: 'close_out'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("close_out", &graph_config_, &outputs);
|
||||
|
||||
CalculatorGraph graph_;
|
||||
@@ -386,7 +386,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
|
||||
protected:
|
||||
void SetUp() override {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'input'
|
||||
input_stream: 'force_main_empty'
|
||||
input_stream: 'force_loop_empty'
|
||||
@@ -424,7 +424,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
|
||||
input_stream: 'passed_through_prev_loop'
|
||||
output_stream: 'passed_through_input_and_prev_loop'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
|
||||
&output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config, {}));
|
||||
@@ -724,7 +724,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'input'
|
||||
# Drops "loop" when set to "true", delaying output of prev_loop, hence
|
||||
# delaying output of the graph.
|
||||
@@ -755,7 +755,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
|
||||
input_stream: 'passed_through_prev_loop'
|
||||
output_stream: 'passed_through_input_and_prev_loop'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
|
||||
&output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config, {}));
|
||||
|
||||
@@ -27,7 +27,7 @@ namespace mediapipe {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
@@ -53,7 +53,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -63,7 +63,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
@@ -80,7 +80,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -90,7 +90,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
@@ -107,7 +107,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -117,7 +117,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
|
||||
min_quantized_value: -1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
@@ -133,7 +133,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -143,7 +143,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
|
||||
min_quantized_value: -64
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f};
|
||||
runner.MutableInputs()
|
||||
@@ -171,7 +171,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -181,7 +181,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
|
||||
min_quantized_value: -64
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> vector = {-65.0f, 65.0f};
|
||||
runner.MutableInputs()
|
||||
|
||||
@@ -70,13 +70,13 @@ std::vector<T> PacketValues(const std::vector<Packet>& packets) {
|
||||
constexpr int kNumImageFrames = 5;
|
||||
constexpr int kNumFinished = 3;
|
||||
CalculatorGraphConfig::Node GetDefaultNode() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "RealTimeFlowLimiterCalculator"
|
||||
input_stream: "raw_frames"
|
||||
input_stream: "FINISHED:finished"
|
||||
input_stream_info: { tag_index: "FINISHED" back_edge: true }
|
||||
output_stream: "gated_frames"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
// Simple test to make sure that the RealTimeFlowLimiterCalculator outputs just
|
||||
@@ -219,7 +219,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
|
||||
// Back-edge "finished" limits processing to one frame in-flight.
|
||||
// The two LambdaCalculators are used to keep certain packet sets in flight.
|
||||
CalculatorGraphConfig InflightGraphConfig() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
input_stream: 'in_2'
|
||||
node {
|
||||
@@ -256,7 +256,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
|
||||
output_stream: 'out_1'
|
||||
output_stream: 'out_2'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
protected:
|
||||
@@ -344,7 +344,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
std::vector<Packet> a_passed;
|
||||
std::vector<Packet> b_passed;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_a'
|
||||
input_stream: 'in_b'
|
||||
input_stream: 'finished'
|
||||
@@ -360,7 +360,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
output_stream: 'in_b_sampled'
|
||||
output_stream: 'ALLOW:allow'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::string allow_cb_name;
|
||||
tool::AddVectorSink("in_a_sampled", &graph_config_, &a_passed);
|
||||
tool::AddVectorSink("in_b_sampled", &graph_config_, &b_passed);
|
||||
@@ -442,7 +442,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
TEST(RealTimeFlowLimiterCalculator, CanConsume) {
|
||||
std::vector<Packet> in_sampled_packets_;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
input_stream: 'finished'
|
||||
node {
|
||||
@@ -455,7 +455,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
|
||||
output_stream: 'in_sampled'
|
||||
output_stream: 'ALLOW:allow'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::string allow_cb_name;
|
||||
tool::AddVectorSink("in_sampled", &graph_config_, &in_sampled_packets_);
|
||||
tool::AddCallbackCalculator("allow", &graph_config_, &allow_cb_name, true);
|
||||
|
||||
@@ -36,7 +36,7 @@ using testing::HasSubstr;
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -45,7 +45,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TICK:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -58,7 +58,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "timestamp"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -67,7 +67,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TIMESTAMP:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -79,7 +79,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -88,7 +88,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "DOES_NOT_EXIST:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -102,7 +102,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
@@ -113,7 +113,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||
output_stream: "AT_TICK:packet0"
|
||||
output_stream: "AT_PRE_STREAM:packet1"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -127,7 +127,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
node {
|
||||
@@ -136,7 +136,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||
output_stream: "AT_PRESTREAM:0:packet0"
|
||||
output_stream: "AT_PRESTREAM:1:packet1"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -149,7 +149,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
node {
|
||||
@@ -158,7 +158,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "AT_PRESTREAM:packet0"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -209,7 +209,7 @@ TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
|
||||
TEST(SidePacketToStreamCalculator, AtTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -219,7 +219,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TICK:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("packet", &graph_config, &output_packets);
|
||||
CalculatorGraph graph;
|
||||
@@ -251,7 +251,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
|
||||
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
@@ -265,7 +265,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||
output_stream: "AT_TICK:0:packet0"
|
||||
output_stream: "AT_TICK:1:packet1"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets0;
|
||||
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
|
||||
std::vector<Packet> output_packets1;
|
||||
@@ -305,7 +305,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||
TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_side_packet: "timestamp"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -315,7 +315,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TIMESTAMP:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("packet", &graph_config, &output_packets);
|
||||
CalculatorGraph graph;
|
||||
@@ -337,7 +337,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||
TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_side_packet: "timestamp"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
@@ -350,7 +350,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
|
||||
output_stream: "AT_TIMESTAMP:0:packet0"
|
||||
output_stream: "AT_TIMESTAMP:1:packet1"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets0;
|
||||
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
|
||||
std::vector<Packet> output_packets1;
|
||||
|
||||
@@ -122,7 +122,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -138,7 +138,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
@@ -171,7 +171,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -183,7 +183,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -196,7 +196,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -209,7 +209,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -223,7 +223,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -238,7 +238,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -252,7 +252,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -266,7 +266,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -281,7 +281,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -298,7 +298,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
@@ -334,7 +334,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -349,7 +349,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
|
||||
@@ -377,7 +377,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -394,7 +394,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
|
||||
@@ -163,7 +163,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -179,7 +179,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
@@ -214,7 +214,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -226,7 +226,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -240,7 +240,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -253,7 +253,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -269,7 +269,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -284,7 +284,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -299,7 +299,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -313,7 +313,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -330,7 +330,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -347,7 +347,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
@@ -385,7 +385,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -400,7 +400,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
|
||||
@@ -428,7 +428,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -445,7 +445,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -511,7 +511,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
@@ -524,7 +524,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -536,7 +536,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
@@ -552,7 +552,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
@@ -592,7 +592,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
@@ -609,7 +609,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
@@ -646,7 +646,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
@@ -661,7 +661,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
|
||||
@@ -41,7 +41,7 @@ constexpr char kWidthTag[] = "WIDTH";
|
||||
TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
|
||||
auto calculator_node =
|
||||
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
|
||||
R"(
|
||||
R"pb(
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_frames"
|
||||
output_stream: "IMAGE_GPU:cropped_output_frames"
|
||||
@@ -54,7 +54,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
|
||||
rotation: 0.3
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
@@ -79,7 +79,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
|
||||
TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
|
||||
auto calculator_node =
|
||||
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
|
||||
R"(
|
||||
R"pb(
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_frames"
|
||||
output_stream: "IMAGE_GPU:cropped_output_frames"
|
||||
@@ -94,7 +94,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
|
||||
rotation: 0.3
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
@@ -119,7 +119,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
|
||||
TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
|
||||
auto calculator_node =
|
||||
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
|
||||
R"(
|
||||
R"pb(
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_frames"
|
||||
input_stream: "WIDTH:crop_width"
|
||||
@@ -136,7 +136,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
|
||||
rotation: 0.3
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
@@ -168,7 +168,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
|
||||
TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
|
||||
auto calculator_node =
|
||||
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
|
||||
R"(
|
||||
R"pb(
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_frames"
|
||||
input_stream: "RECT:rect"
|
||||
@@ -184,7 +184,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
|
||||
rotation: 0.3
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
@@ -196,9 +196,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
|
||||
calculator_state.get(), inputTags, tool::CreateTagMap({}).value());
|
||||
auto& inputs = cc->Inputs();
|
||||
mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
|
||||
R"(
|
||||
R"pb(
|
||||
width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
|
||||
)");
|
||||
)pb");
|
||||
inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect);
|
||||
RectSpec expectRect = {
|
||||
.width = 1,
|
||||
|
||||
@@ -50,11 +50,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInStreams) {
|
||||
MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ImageFilePropertiesCalculator"
|
||||
input_stream: "image_bytes"
|
||||
output_stream: "properties"
|
||||
)");
|
||||
)pb");
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
@@ -79,11 +79,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInSidePackets) {
|
||||
MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ImageFilePropertiesCalculator"
|
||||
input_side_packet: "image_bytes"
|
||||
output_side_packet: "properties"
|
||||
)");
|
||||
)pb");
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Index(0) =
|
||||
@@ -108,11 +108,11 @@ TEST(ImageFilePropertiesCalculatorTest,
|
||||
MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ImageFilePropertiesCalculator"
|
||||
input_stream: "image_bytes"
|
||||
output_side_packet: "properties"
|
||||
)");
|
||||
)pb");
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
|
||||
@@ -36,11 +36,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestRgbJpeg) {
|
||||
Packet input_packet = MakePacket<std::string>(contents);
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "OpenCvEncodedImageToImageFrameCalculator"
|
||||
input_stream: "encoded_image"
|
||||
output_stream: "image_frame"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
input_packet.At(Timestamp(0)));
|
||||
@@ -79,11 +79,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestGrayscaleJpeg) {
|
||||
reinterpret_cast<const char*>(&encode_buffer[0]), encode_buffer.size())));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "OpenCvEncodedImageToImageFrameCalculator"
|
||||
input_stream: "encoded_image"
|
||||
output_stream: "image_frame"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
input_packet.At(Timestamp(0)));
|
||||
|
||||
@@ -561,13 +561,13 @@ cc_test(
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
@@ -28,7 +29,6 @@
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
|
||||
@@ -41,7 +41,6 @@ class InferenceCalculatorSelectorImpl
|
||||
(options.has_delegate() && options.delegate().has_gpu());
|
||||
if (should_use_gpu) {
|
||||
impls.emplace_back("Metal");
|
||||
impls.emplace_back("MlDrift");
|
||||
impls.emplace_back("Gl");
|
||||
}
|
||||
impls.emplace_back("Cpu");
|
||||
|
||||
@@ -118,10 +118,6 @@ struct InferenceCalculatorGl : public InferenceCalculator {
|
||||
static constexpr char kCalculatorName[] = "InferenceCalculatorGl";
|
||||
};
|
||||
|
||||
struct InferenceCalculatorMlDrift : public InferenceCalculator {
|
||||
static constexpr char kCalculatorName[] = "InferenceCalculatorMlDrift";
|
||||
};
|
||||
|
||||
struct InferenceCalculatorMetal : public InferenceCalculator {
|
||||
static constexpr char kCalculatorName[] = "InferenceCalculatorMetal";
|
||||
};
|
||||
|
||||
@@ -51,12 +51,12 @@ message InferenceCalculatorOptions {
|
||||
|
||||
// This option is valid for TFLite GPU delegate API2 only,
|
||||
// Choose any of available APIs to force running inference using it.
|
||||
enum API {
|
||||
enum Api {
|
||||
ANY = 0;
|
||||
OPENGL = 1;
|
||||
OPENCL = 2;
|
||||
}
|
||||
optional API api = 4 [default = ANY];
|
||||
optional Api api = 4 [default = ANY];
|
||||
|
||||
// This option is valid for TFLite GPU delegate API2 only,
|
||||
// Set to true to use 16-bit float precision. If max precision is needed,
|
||||
|
||||
@@ -136,7 +136,7 @@ absl::Status InferenceCalculatorCpuImpl::LoadModel(CalculatorContext* cc) {
|
||||
const auto& model = *model_packet_.Get();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver =
|
||||
kSideInCustomOpResolver(cc).GetOr(
|
||||
tflite::ops::builtin::BuiltinOpResolver());
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
|
||||
|
||||
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
|
||||
RET_CHECK(interpreter_);
|
||||
|
||||
@@ -57,11 +57,7 @@ const std::vector<Param>& GetParams() {
|
||||
// Metal is not available on the iOS simulator.
|
||||
p.push_back({"Metal", "Metal"});
|
||||
p.back().delegate.mutable_gpu();
|
||||
#endif // TARGET_IPHONE_SIMULATOR
|
||||
#if __EMSCRIPTEN__
|
||||
p.push_back({"MlDrift", "MlDrift"});
|
||||
p.back().delegate.mutable_gpu();
|
||||
#endif // __EMSCRIPTEN__
|
||||
#endif // TARGET_IPHONE_SIMULATOR
|
||||
#if __ANDROID__ && 0 // Disabled for now since emulator can't go GLESv3
|
||||
p.push_back({"Gl", "Gl"});
|
||||
p.back().delegate.mutable_gpu();
|
||||
@@ -78,18 +74,6 @@ const std::vector<Param>& GetParams() {
|
||||
|
||||
class InferenceCalculatorTest : public testing::TestWithParam<Param> {
|
||||
protected:
|
||||
#if __EMSCRIPTEN__
|
||||
// TODO: fix Tensor locking.
|
||||
// The MlDrift backend currently fails in debug mode without this,
|
||||
// because of Tensor locking issues. I am adding this temporarily since
|
||||
// the calculator is already being used and it's better to have test
|
||||
// coverage for it. Also, the issue doesn't apply to our Emscripten
|
||||
// build in practice since it's single-threaded.
|
||||
void SetUp(void) override {
|
||||
absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kIgnore);
|
||||
}
|
||||
#endif // __EMSCRIPTEN__
|
||||
|
||||
void SetDelegateForParam(mediapipe::CalculatorGraphConfig_Node* node) {
|
||||
*node->mutable_options()
|
||||
->MutableExtension(mediapipe::InferenceCalculatorOptions::ext)
|
||||
|
||||
@@ -63,7 +63,7 @@ class InferenceCalculatorGlImpl
|
||||
mediapipe::GlCalculatorHelper gpu_helper_;
|
||||
std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
|
||||
bool allow_precision_loss_ = false;
|
||||
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::API
|
||||
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
|
||||
tflite_gpu_runner_api_;
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
|
||||
@@ -244,7 +244,7 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
|
||||
const auto& model = *model_packet_.Get();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver =
|
||||
kSideInCustomOpResolver(cc).GetOr(
|
||||
tflite::ops::builtin::BuiltinOpResolver());
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
|
||||
|
||||
// Create runner
|
||||
tflite::gpu::InferenceOptions options;
|
||||
@@ -294,7 +294,7 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
|
||||
const auto& model = *model_packet_.Get();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver =
|
||||
kSideInCustomOpResolver(cc).GetOr(
|
||||
tflite::ops::builtin::BuiltinOpResolver());
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
|
||||
|
||||
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
|
||||
RET_CHECK(interpreter_);
|
||||
|
||||
@@ -200,7 +200,7 @@ absl::Status InferenceCalculatorMetalImpl::LoadModel(CalculatorContext* cc) {
|
||||
const auto& model = *model_packet_.Get();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver =
|
||||
kSideInCustomOpResolver(cc).GetOr(
|
||||
tflite::ops::builtin::BuiltinOpResolver());
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
|
||||
|
||||
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
|
||||
RET_CHECK(interpreter_);
|
||||
|
||||
@@ -84,7 +84,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
|
||||
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
@@ -96,7 +96,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
@@ -146,7 +146,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
|
||||
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
@@ -158,7 +158,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
@@ -205,7 +205,7 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
|
||||
CalculatorGraph graph;
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "input_image"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
@@ -220,7 +220,7 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
|
||||
@@ -56,14 +56,14 @@ class TensorsToClassificationCalculatorTest : public ::testing::Test {
|
||||
};
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -85,7 +85,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) {
|
||||
}
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
@@ -94,7 +94,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
|
||||
label_map_path: "mediapipe/calculators/tensor/testdata/labelmap.txt"
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -117,7 +117,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest,
|
||||
CorrectOutputWithLabelMinScoreThreshold) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
@@ -126,7 +126,7 @@ TEST_F(TensorsToClassificationCalculatorTest,
|
||||
min_score_threshold: 0.6
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -144,14 +144,14 @@ TEST_F(TensorsToClassificationCalculatorTest,
|
||||
}
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] { top_k: 2 }
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
@@ -57,11 +57,11 @@ class TensorsToFloatsCalculatorTest : public ::testing::Test {
|
||||
};
|
||||
|
||||
TEST_F(TensorsToFloatsCalculatorTest, SingleValue) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "FLOAT:float"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
const float single_value = 0.5;
|
||||
BuildGraph(&runner, {single_value});
|
||||
@@ -76,11 +76,11 @@ TEST_F(TensorsToFloatsCalculatorTest, SingleValue) {
|
||||
}
|
||||
|
||||
TEST_F(TensorsToFloatsCalculatorTest, SingleValueAsVector) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "FLOATS:floats"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
const float single_value = 0.5;
|
||||
BuildGraph(&runner, {single_value});
|
||||
@@ -95,11 +95,11 @@ TEST_F(TensorsToFloatsCalculatorTest, SingleValueAsVector) {
|
||||
}
|
||||
|
||||
TEST_F(TensorsToFloatsCalculatorTest, FloatVector) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "FLOATS:floats"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
const std::vector<float> input_values = {0.f, 0.5f, 1.0f};
|
||||
BuildGraph(&runner, input_values);
|
||||
@@ -116,14 +116,14 @@ TEST_F(TensorsToFloatsCalculatorTest, FloatVector) {
|
||||
}
|
||||
|
||||
TEST_F(TensorsToFloatsCalculatorTest, FloatVectorWithSigmoid) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "FLOATS:floats"
|
||||
options {
|
||||
[mediapipe.TensorsToFloatsCalculatorOptions.ext] { activation: SIGMOID }
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
const std::vector<float> input_values = {-1.f, 0.f, 1.0f};
|
||||
const std::vector<float> expected_output_with_sigmoid = {0.269f, 0.5f,
|
||||
|
||||
@@ -892,13 +892,13 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:tag_map_helper",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
@@ -923,13 +923,13 @@ cc_test(
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework:packet_generator_cc_proto",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:tag_map_helper",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
@@ -954,11 +954,11 @@ cc_test(
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework:packet_generator_cc_proto",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:tag_map_helper",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:all_kernels",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
@@ -981,11 +981,11 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:tag_map_helper",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:all_kernels",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
@@ -1144,8 +1144,8 @@ cc_test(
|
||||
":tensorflow_inference_calculator",
|
||||
":tensorflow_session_from_frozen_graph_generator",
|
||||
":tensorflow_session_from_frozen_graph_generator_cc_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
|
||||
+8
-8
@@ -68,31 +68,31 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
|
||||
|
||||
void CreateNodeConfig(CalculatorGraphConfig::Node* node_config) const {
|
||||
ASSERT_NE(nullptr, node_config);
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ObjectDetectionTensorsToDetectionsCalculator"
|
||||
input_stream: "NUM_DETECTIONS:num_detections"
|
||||
input_stream: "BOXES:boxes"
|
||||
input_stream: "SCORES:scores"
|
||||
input_stream: "CLASSES:classes"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
void CreateNodeConfigRawTensors(
|
||||
CalculatorGraphConfig::Node* node_config) const {
|
||||
ASSERT_NE(nullptr, node_config);
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ObjectDetectionTensorsToDetectionsCalculator"
|
||||
input_stream: "BOXES:raw_detection_boxes"
|
||||
input_stream: "SCORES:raw_detection_scores"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
void CreateNodeConfigWithKeypoints(
|
||||
CalculatorGraphConfig::Node* node_config) const {
|
||||
ASSERT_NE(nullptr, node_config);
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ObjectDetectionTensorsToDetectionsCalculator"
|
||||
input_stream: "NUM_DETECTIONS:num_detections"
|
||||
input_stream: "BOXES:boxes"
|
||||
@@ -100,7 +100,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
|
||||
input_stream: "CLASSES:classes"
|
||||
input_stream: "KEYPOINTS:keypoints"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
void SetUpCalculatorRunner() {
|
||||
@@ -177,7 +177,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
|
||||
InsertExtraSingltonDim(&input_scores_);
|
||||
InsertExtraSingltonDim(&input_classes_);
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ObjectDetectionTensorsToDetectionsCalculator"
|
||||
input_stream: "NUM_DETECTIONS:num_detections"
|
||||
input_stream: "BOXES:boxes"
|
||||
@@ -188,7 +188,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
|
||||
[mediapipe.ObjectDetectionsTensorToDetectionsCalculatorOptions
|
||||
.ext]: { tensor_dim_to_squeeze: 0 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
runner_ = absl::make_unique<CalculatorRunner>(node_config);
|
||||
runner_->MutableInputs()
|
||||
->Tag(kNumDetections)
|
||||
|
||||
@@ -267,6 +267,17 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
}
|
||||
}
|
||||
|
||||
absl::Status VerifySize() {
|
||||
const int64 MAX_PROTO_BYTES = 1073741823;
|
||||
std::string id = mpms::HasExampleId(*sequence_)
|
||||
? mpms::GetExampleId(*sequence_)
|
||||
: "example";
|
||||
RET_CHECK_LT(sequence_->ByteSizeLong(), MAX_PROTO_BYTES)
|
||||
<< "sequence '" << id
|
||||
<< "' would be too many bytes to serialize after adding features.";
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Close(CalculatorContext* cc) override {
|
||||
auto& options = cc->Options<PackMediaSequenceCalculatorOptions>();
|
||||
if (options.reconcile_metadata()) {
|
||||
@@ -275,6 +286,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
options.reconcile_region_annotations(), sequence_.get()));
|
||||
}
|
||||
|
||||
if (options.skip_large_sequences()) {
|
||||
RET_CHECK_OK(VerifySize());
|
||||
}
|
||||
if (options.output_only_if_all_present()) {
|
||||
absl::Status status = VerifySequence();
|
||||
if (!status.ok()) {
|
||||
|
||||
@@ -61,4 +61,8 @@ message PackMediaSequenceCalculatorOptions {
|
||||
// present, the previous images and timestamps will be removed before adding
|
||||
// the new images.
|
||||
optional bool replace_data_instead_of_append = 4 [default = true];
|
||||
|
||||
// If true, will return an error status if an output sequence would be too
|
||||
// many bytes to serialize.
|
||||
optional bool skip_large_sequences = 7 [default = true];
|
||||
}
|
||||
|
||||
@@ -889,5 +889,24 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
}
|
||||
|
||||
TEST_F(PackMediaSequenceCalculatorTest, TestTooLargeInputFailsSoftly) {
|
||||
SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true);
|
||||
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
|
||||
|
||||
// 1 billion floats should be > 1GB which can't be serialized. It should fail
|
||||
// gracefully with this input.
|
||||
int num_timesteps = 1000;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto vf_ptr = ::absl::make_unique<std::vector<float>>(1000000, i);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FLOAT_FEATURE_TEST")
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
Adopt(input_sequence.release());
|
||||
ASSERT_FALSE(runner_->Run().ok());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -16,12 +16,12 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
|
||||
+1
-1
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
|
||||
@@ -19,7 +20,6 @@
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
|
||||
+1
-1
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
|
||||
@@ -19,7 +20,6 @@
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/packet_generator.pb.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
|
||||
+1
-1
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/strings/str_replace.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
|
||||
@@ -20,7 +21,6 @@
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
|
||||
+1
-1
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/strings/str_replace.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
|
||||
@@ -19,7 +20,6 @@
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/packet_generator.pb.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/container/flat_hash_map.h"
|
||||
#include "absl/strings/match.h"
|
||||
#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.pb.h"
|
||||
|
||||
@@ -56,11 +56,4 @@ message UnpackMediaSequenceCalculatorOptions {
|
||||
// the clip start and end times and outputs these for the
|
||||
// AudioDecoderCalculator to consume.
|
||||
optional AudioDecoderOptions base_audio_decoder_options = 9;
|
||||
|
||||
optional string keypoint_names = 10 [
|
||||
default =
|
||||
"NOSE,LEFT_EAR,RIGHT_EAR,LEFT_SHOULDER,RIGHT_SHOULDER,LEFT_FORE_PAW,RIGHT_FORE_PAW,LEFT_HIP,RIGHT_HIP,LEFT_HIND_PAW,RIGHT_HIND_PAW,ROOT_TAIL"
|
||||
];
|
||||
// When the keypoint doesn't exists, output this default value.
|
||||
optional float default_keypoint_location = 11 [default = -1.0];
|
||||
}
|
||||
|
||||
@@ -147,11 +147,11 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -12,11 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/formats/object_detection/anchor.pb.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
@@ -68,7 +68,7 @@ void CompareAnchors(const std::vector<Anchor>& anchors_0,
|
||||
}
|
||||
|
||||
TEST(SsdAnchorCalculatorTest, FaceDetectionConfig) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
options {
|
||||
@@ -89,7 +89,7 @@ TEST(SsdAnchorCalculatorTest, FaceDetectionConfig) {
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
|
||||
|
||||
@@ -106,7 +106,7 @@ TEST(SsdAnchorCalculatorTest, FaceDetectionConfig) {
|
||||
}
|
||||
|
||||
TEST(SsdAnchorCalculatorTest, MobileSSDConfig) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
options {
|
||||
@@ -132,7 +132,7 @@ TEST(SsdAnchorCalculatorTest, MobileSSDConfig) {
|
||||
reduce_boxes_in_lowest_layer: true
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
|
||||
const auto& anchors =
|
||||
|
||||
@@ -85,7 +85,7 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixColMajor) {
|
||||
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
@@ -97,7 +97,7 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixColMajor) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
@@ -146,7 +146,7 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixRowMajor) {
|
||||
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
@@ -158,7 +158,7 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixRowMajor) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
@@ -204,7 +204,7 @@ TEST_F(TfLiteConverterCalculatorTest, CustomDivAndSub) {
|
||||
CalculatorGraph graph;
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "input_image"
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
@@ -219,7 +219,7 @@ TEST_F(TfLiteConverterCalculatorTest, CustomDivAndSub) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
|
||||
@@ -278,7 +278,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
|
||||
|
||||
bool use_advanced_gpu_api_ = false;
|
||||
bool allow_precision_loss_ = false;
|
||||
mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::API
|
||||
mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::Api
|
||||
tflite_gpu_runner_api_;
|
||||
|
||||
bool use_kernel_caching_ = false;
|
||||
@@ -702,11 +702,16 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
|
||||
#if MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc));
|
||||
const auto& model = *model_packet_.Get<TfLiteModelPtr>();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver;
|
||||
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates
|
||||
default_op_resolver;
|
||||
auto op_resolver_ptr =
|
||||
static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
|
||||
&default_op_resolver);
|
||||
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
|
||||
op_resolver = cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Get<tflite::ops::builtin::BuiltinOpResolver>();
|
||||
op_resolver_ptr = &(cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Get<tflite::ops::builtin::BuiltinOpResolver>());
|
||||
}
|
||||
|
||||
// Create runner
|
||||
@@ -733,7 +738,7 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
|
||||
}
|
||||
}
|
||||
MP_RETURN_IF_ERROR(
|
||||
tflite_gpu_runner_->InitializeWithModel(model, op_resolver));
|
||||
tflite_gpu_runner_->InitializeWithModel(model, *op_resolver_ptr));
|
||||
|
||||
// Allocate interpreter memory for cpu output.
|
||||
if (!gpu_output_) {
|
||||
@@ -786,18 +791,24 @@ absl::Status TfLiteInferenceCalculator::LoadModel(CalculatorContext* cc) {
|
||||
|
||||
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc));
|
||||
const auto& model = *model_packet_.Get<TfLiteModelPtr>();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver;
|
||||
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates
|
||||
default_op_resolver;
|
||||
auto op_resolver_ptr =
|
||||
static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
|
||||
&default_op_resolver);
|
||||
|
||||
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
|
||||
op_resolver = cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Get<tflite::ops::builtin::BuiltinOpResolver>();
|
||||
op_resolver_ptr = &(cc->InputSidePackets()
|
||||
.Tag("CUSTOM_OP_RESOLVER")
|
||||
.Get<tflite::ops::builtin::BuiltinOpResolver>());
|
||||
}
|
||||
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
interpreter_ =
|
||||
BuildEdgeTpuInterpreter(model, &op_resolver, edgetpu_context_.get());
|
||||
BuildEdgeTpuInterpreter(model, op_resolver_ptr, edgetpu_context_.get());
|
||||
#else
|
||||
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
|
||||
tflite::InterpreterBuilder(model, *op_resolver_ptr)(&interpreter_);
|
||||
#endif // MEDIAPIPE_EDGE_TPU
|
||||
|
||||
RET_CHECK(interpreter_);
|
||||
|
||||
@@ -51,12 +51,12 @@ message TfLiteInferenceCalculatorOptions {
|
||||
|
||||
// This option is valid for TFLite GPU delegate API2 only,
|
||||
// Choose any of available APIs to force running inference using it.
|
||||
enum API {
|
||||
enum Api {
|
||||
ANY = 0;
|
||||
OPENGL = 1;
|
||||
OPENCL = 2;
|
||||
}
|
||||
optional API api = 4 [default = ANY];
|
||||
optional Api api = 4 [default = ANY];
|
||||
|
||||
// This option is valid for TFLite GPU delegate API2 only,
|
||||
// Set to true to use 16-bit float precision. If max precision is needed,
|
||||
|
||||
@@ -28,7 +28,7 @@ TEST(TfLiteModelCalculatorTest, SmokeTest) {
|
||||
// Prepare single calculator graph to and wait for packets.
|
||||
CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
|
||||
CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:model_path"
|
||||
@@ -52,7 +52,7 @@ TEST(TfLiteModelCalculatorTest, SmokeTest) {
|
||||
input_side_packet: "MODEL_BLOB:model_blob"
|
||||
output_side_packet: "MODEL:model"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph(graph_config);
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
@@ -73,14 +73,14 @@ class TfLiteTensorsToClassificationCalculatorTest : public ::testing::Test {
|
||||
};
|
||||
|
||||
TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutput) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TfLiteTensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -103,7 +103,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutput) {
|
||||
|
||||
TEST_F(TfLiteTensorsToClassificationCalculatorTest,
|
||||
CorrectOutputWithLabelMapPath) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TfLiteTensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
@@ -112,7 +112,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest,
|
||||
label_map_path: "mediapipe/calculators/tflite/testdata/labelmap.txt"
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -135,7 +135,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest,
|
||||
|
||||
TEST_F(TfLiteTensorsToClassificationCalculatorTest,
|
||||
CorrectOutputWithLabelMinScoreThreshold) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TfLiteTensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
@@ -144,7 +144,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest,
|
||||
min_score_threshold: 0.6
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -162,7 +162,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest,
|
||||
}
|
||||
|
||||
TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TfLiteTensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
@@ -171,7 +171,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
|
||||
top_k: 2
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
@@ -841,12 +841,39 @@ cc_library(
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util/filtering:one_euro_filter",
|
||||
"//mediapipe/util/filtering:relative_velocity_filter",
|
||||
"@com_google_absl//absl/algorithm:container",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "visibility_smoothing_calculator_proto",
|
||||
srcs = ["visibility_smoothing_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "visibility_smoothing_calculator",
|
||||
srcs = ["visibility_smoothing_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":visibility_smoothing_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util/filtering:low_pass_filter",
|
||||
"@com_google_absl//absl/algorithm:container",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "landmarks_to_floats_calculator",
|
||||
srcs = ["landmarks_to_floats_calculator.cc"],
|
||||
@@ -858,7 +885,7 @@ cc_library(
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -1194,3 +1221,34 @@ cc_library(
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "detection_classifications_merger_calculator",
|
||||
srcs = ["detection_classifications_merger_calculator.cc"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "detection_classifications_merger_calculator_test",
|
||||
srcs = ["detection_classifications_merger_calculator_test.cc"],
|
||||
deps = [
|
||||
":detection_classifications_merger_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:message_matchers",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -97,7 +97,7 @@ class AssociationDetectionCalculatorTest : public ::testing::Test {
|
||||
};
|
||||
|
||||
TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AssociationDetectionCalculator"
|
||||
input_stream: "input_vec_0"
|
||||
input_stream: "input_vec_1"
|
||||
@@ -108,7 +108,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) {
|
||||
min_similarity_threshold: 0.1
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// Input Stream 0: det_0, det_1, det_2.
|
||||
auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
|
||||
@@ -160,7 +160,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) {
|
||||
}
|
||||
|
||||
TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AssociationDetectionCalculator"
|
||||
input_stream: "PREV:input_vec_0"
|
||||
input_stream: "input_vec_1"
|
||||
@@ -170,7 +170,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) {
|
||||
min_similarity_threshold: 0.1
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// Input Stream 0: det_3, det_4.
|
||||
auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
|
||||
@@ -209,7 +209,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) {
|
||||
}
|
||||
|
||||
TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestReverse) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AssociationDetectionCalculator"
|
||||
input_stream: "input_vec_0"
|
||||
input_stream: "input_vec_1"
|
||||
@@ -220,7 +220,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestReverse) {
|
||||
min_similarity_threshold: 0.1
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// Input Stream 0: det_5.
|
||||
auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
|
||||
@@ -332,7 +332,7 @@ class AssociationNormRectCalculatorTest : public ::testing::Test {
|
||||
};
|
||||
|
||||
TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "input_vec_0"
|
||||
input_stream: "input_vec_1"
|
||||
@@ -343,7 +343,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) {
|
||||
min_similarity_threshold: 0.1
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// Input Stream 0: nr_0, nr_1, nr_2.
|
||||
auto input_vec_0 =
|
||||
@@ -386,7 +386,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) {
|
||||
}
|
||||
|
||||
TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "input_vec_0"
|
||||
input_stream: "input_vec_1"
|
||||
@@ -397,7 +397,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) {
|
||||
min_similarity_threshold: 0.1
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// Input Stream 0: nr_5.
|
||||
auto input_vec_0 =
|
||||
@@ -441,7 +441,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) {
|
||||
}
|
||||
|
||||
TEST_F(AssociationNormRectCalculatorTest, NormRectAssocSingleInputStream) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "input_vec"
|
||||
output_stream: "output_vec"
|
||||
@@ -450,7 +450,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocSingleInputStream) {
|
||||
min_similarity_threshold: 0.1
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// Input Stream : nr_3, nr_5.
|
||||
auto input_vec =
|
||||
|
||||
@@ -41,14 +41,14 @@ void AddInputVector(const std::vector<int>& input, int64 timestamp,
|
||||
|
||||
TEST(TestIntCollectionHasMinSizeCalculator, DoesHaveMinSize) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestIntCollectionHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.CollectionHasMinSizeCalculatorOptions.ext] { min_size: 2 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
|
||||
@@ -70,12 +70,12 @@ TEST(TestIntCollectionHasMinSizeCalculator, DoesHaveMinSize) {
|
||||
TEST(TestIntCollectionHasMinSizeCalculator,
|
||||
DoesHaveMinSize_MinSizeAsSidePacket) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestIntCollectionHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:input_vector"
|
||||
input_side_packet: "min_size"
|
||||
output_stream: "output_vector"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
|
||||
@@ -98,14 +98,14 @@ TEST(TestIntCollectionHasMinSizeCalculator,
|
||||
|
||||
TEST(TestIntCollectionHasMinSizeCalculator, DoesNotHaveMinSize) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestIntCollectionHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.CollectionHasMinSizeCalculatorOptions.ext] { min_size: 3 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
|
||||
@@ -127,12 +127,12 @@ TEST(TestIntCollectionHasMinSizeCalculator, DoesNotHaveMinSize) {
|
||||
TEST(TestIntCollectionHasMinSizeCalculator,
|
||||
DoesNotHaveMinSize_MinSizeAsSidePacket) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestIntCollectionHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:input_vector"
|
||||
input_side_packet: "min_size"
|
||||
output_stream: "output_vector"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
namespace {} // namespace
|
||||
|
||||
// Replaces the classification labels and scores from the input `Detection` with
|
||||
// the ones provided into the input `ClassificationList`. Namely:
|
||||
// * `label_id[i]` becomes `classification[i].index`
|
||||
// * `score[i]` becomes `classification[i].score`
|
||||
// * `label[i]` becomes `classification[i].label` (if present)
|
||||
//
|
||||
// In case the input `ClassificationList` contains no results (i.e.
|
||||
// `classification` is empty, which may happen if the classifier uses a score
|
||||
// threshold and no confident enough result were returned), the input
|
||||
// `Detection` is returned unchanged.
|
||||
//
|
||||
// This is specifically designed for two-stage detection cascades where the
|
||||
// detections returned by a standalone detector (typically a class-agnostic
|
||||
// localizer) are fed e.g. into a `TfLiteTaskImageClassifierCalculator` through
|
||||
// the optional "RECT" or "NORM_RECT" input, e.g:
|
||||
//
|
||||
// node {
|
||||
// calculator: "DetectionsToRectsCalculator"
|
||||
// # Output of an upstream object detector.
|
||||
// input_stream: "DETECTION:detection"
|
||||
// output_stream: "NORM_RECT:norm_rect"
|
||||
// }
|
||||
// node {
|
||||
// calculator: "TfLiteTaskImageClassifierCalculator"
|
||||
// input_stream: "IMAGE:image"
|
||||
// input_stream: "NORM_RECT:norm_rect"
|
||||
// output_stream: "CLASSIFICATION_RESULT:classification_result"
|
||||
// }
|
||||
// node {
|
||||
// calculator: "TfLiteTaskClassificationResultToClassificationsCalculator"
|
||||
// input_stream: "CLASSIFICATION_RESULT:classification_result"
|
||||
// output_stream: "CLASSIFICATION_LIST:classification_list"
|
||||
// }
|
||||
// node {
|
||||
// calculator: "DetectionClassificationsMergerCalculator"
|
||||
// input_stream: "INPUT_DETECTION:detection"
|
||||
// input_stream: "CLASSIFICATION_LIST:classification_list"
|
||||
// # Final output.
|
||||
// output_stream: "OUTPUT_DETECTION:classified_detection"
|
||||
// }
|
||||
//
|
||||
// Inputs:
|
||||
// INPUT_DETECTION: `Detection` proto.
|
||||
// CLASSIFICATION_LIST: `ClassificationList` proto.
|
||||
//
|
||||
// Output:
|
||||
// OUTPUT_DETECTION: modified `Detection` proto.
|
||||
class DetectionClassificationsMergerCalculator : public Node {
|
||||
public:
|
||||
static constexpr Input<Detection> kInputDetection{"INPUT_DETECTION"};
|
||||
static constexpr Input<ClassificationList> kClassificationList{
|
||||
"CLASSIFICATION_LIST"};
|
||||
static constexpr Output<Detection> kOutputDetection{"OUTPUT_DETECTION"};
|
||||
|
||||
MEDIAPIPE_NODE_CONTRACT(kInputDetection, kClassificationList,
|
||||
kOutputDetection);
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
MEDIAPIPE_REGISTER_NODE(DetectionClassificationsMergerCalculator);
|
||||
|
||||
absl::Status DetectionClassificationsMergerCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
if (kInputDetection(cc).IsEmpty() && kClassificationList(cc).IsEmpty()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
RET_CHECK(!kInputDetection(cc).IsEmpty());
|
||||
RET_CHECK(!kClassificationList(cc).IsEmpty());
|
||||
|
||||
Detection detection = *kInputDetection(cc);
|
||||
const ClassificationList& classification_list = *kClassificationList(cc);
|
||||
|
||||
// Update input detection only if classification did return results.
|
||||
if (classification_list.classification_size() != 0) {
|
||||
detection.clear_label_id();
|
||||
detection.clear_score();
|
||||
detection.clear_label();
|
||||
detection.clear_display_name();
|
||||
for (const auto& classification : classification_list.classification()) {
|
||||
if (!classification.has_index()) {
|
||||
return absl::InvalidArgumentError(
|
||||
"Missing required 'index' field in Classification proto.");
|
||||
}
|
||||
detection.add_label_id(classification.index());
|
||||
if (!classification.has_score()) {
|
||||
return absl::InvalidArgumentError(
|
||||
"Missing required 'score' field in Classification proto.");
|
||||
}
|
||||
detection.add_score(classification.score());
|
||||
if (classification.has_label()) {
|
||||
detection.add_label(classification.label());
|
||||
}
|
||||
if (classification.has_display_name()) {
|
||||
detection.add_display_name(classification.display_name());
|
||||
}
|
||||
}
|
||||
// Post-conversion sanity checks.
|
||||
if (detection.label_size() != 0 &&
|
||||
detection.label_size() != detection.label_id_size()) {
|
||||
return absl::InvalidArgumentError(absl::Substitute(
|
||||
"Each input Classification is expected to either always or never "
|
||||
"provide a 'label' field. Found $0 'label' fields for $1 "
|
||||
"'Classification' objects.",
|
||||
/*$0=*/detection.label_size(), /*$1=*/detection.label_id_size()));
|
||||
}
|
||||
if (detection.display_name_size() != 0 &&
|
||||
detection.display_name_size() != detection.label_id_size()) {
|
||||
return absl::InvalidArgumentError(absl::Substitute(
|
||||
"Each input Classification is expected to either always or never "
|
||||
"provide a 'display_name' field. Found $0 'display_name' fields for "
|
||||
"$1 'Classification' objects.",
|
||||
/*$0=*/detection.display_name_size(),
|
||||
/*$1=*/detection.label_id_size()));
|
||||
}
|
||||
}
|
||||
kOutputDetection(cc).Send(detection);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,320 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/message_matchers.h"
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
constexpr char kGraphConfig[] = R"(
|
||||
input_stream: "input_detection"
|
||||
input_stream: "classification_list"
|
||||
output_stream: "output_detection"
|
||||
node {
|
||||
calculator: "DetectionClassificationsMergerCalculator"
|
||||
input_stream: "INPUT_DETECTION:input_detection"
|
||||
input_stream: "CLASSIFICATION_LIST:classification_list"
|
||||
output_stream: "OUTPUT_DETECTION:output_detection"
|
||||
}
|
||||
)";
|
||||
|
||||
constexpr char kInputDetection[] = R"(
|
||||
label: "entity"
|
||||
label_id: 1
|
||||
score: 0.9
|
||||
location_data {
|
||||
format: BOUNDING_BOX
|
||||
bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 }
|
||||
}
|
||||
display_name: "Entity"
|
||||
)";
|
||||
|
||||
// Checks that the input Detection is returned unchanged if the input
|
||||
// ClassificationList does not contain any result.
|
||||
TEST(DetectionClassificationsMergerCalculator, SucceedsWithNoClassification) {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
|
||||
|
||||
// Prepare input packets.
|
||||
const Detection& input_detection =
|
||||
ParseTextProtoOrDie<Detection>(kInputDetection);
|
||||
Packet input_detection_packet =
|
||||
MakePacket<Detection>(input_detection).At(Timestamp(0));
|
||||
const ClassificationList& classification_list =
|
||||
ParseTextProtoOrDie<ClassificationList>("");
|
||||
Packet classification_list_packet =
|
||||
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
|
||||
|
||||
// Catch output.
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(
|
||||
graph.AddPacketToInputStream("input_detection", input_detection_packet));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
|
||||
classification_list_packet));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
// Get and validate output.
|
||||
EXPECT_THAT(output_packets, testing::SizeIs(1));
|
||||
const Detection& output_detection = output_packets[0].Get<Detection>();
|
||||
EXPECT_THAT(output_detection, mediapipe::EqualsProto(input_detection));
|
||||
}
|
||||
|
||||
// Checks that merging succeeds when the input ClassificationList includes
|
||||
// labels and display names.
|
||||
TEST(DetectionClassificationsMergerCalculator,
|
||||
SucceedsWithLabelsAndDisplayNames) {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
|
||||
|
||||
// Prepare input packets.
|
||||
const Detection& input_detection =
|
||||
ParseTextProtoOrDie<Detection>(kInputDetection);
|
||||
Packet input_detection_packet =
|
||||
MakePacket<Detection>(input_detection).At(Timestamp(0));
|
||||
const ClassificationList& classification_list =
|
||||
ParseTextProtoOrDie<ClassificationList>(R"pb(
|
||||
classification { index: 11 score: 0.5 label: "dog" display_name: "Dog" }
|
||||
classification { index: 12 score: 0.4 label: "fox" display_name: "Fox" }
|
||||
)pb");
|
||||
Packet classification_list_packet =
|
||||
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
|
||||
|
||||
// Catch output.
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(
|
||||
graph.AddPacketToInputStream("input_detection", input_detection_packet));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
|
||||
classification_list_packet));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
// Get and validate output.
|
||||
EXPECT_THAT(output_packets, testing::SizeIs(1));
|
||||
const Detection& output_detection = output_packets[0].Get<Detection>();
|
||||
EXPECT_THAT(output_detection,
|
||||
mediapipe::EqualsProto(ParseTextProtoOrDie<Detection>(R"pb(
|
||||
label: "dog"
|
||||
label: "fox"
|
||||
label_id: 11
|
||||
label_id: 12
|
||||
score: 0.5
|
||||
score: 0.4
|
||||
location_data {
|
||||
format: BOUNDING_BOX
|
||||
bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 }
|
||||
}
|
||||
display_name: "Dog"
|
||||
display_name: "Fox"
|
||||
)pb")));
|
||||
}
|
||||
|
||||
// Checks that merging succeeds when the input ClassificationList doesn't
|
||||
// include labels and display names.
|
||||
TEST(DetectionClassificationsMergerCalculator,
|
||||
SucceedsWithoutLabelsAndDisplayNames) {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
|
||||
|
||||
// Prepare input packets.
|
||||
const Detection& input_detection =
|
||||
ParseTextProtoOrDie<Detection>(kInputDetection);
|
||||
Packet input_detection_packet =
|
||||
MakePacket<Detection>(input_detection).At(Timestamp(0));
|
||||
const ClassificationList& classification_list =
|
||||
ParseTextProtoOrDie<ClassificationList>(R"pb(
|
||||
classification { index: 11 score: 0.5 }
|
||||
classification { index: 12 score: 0.4 }
|
||||
)pb");
|
||||
Packet classification_list_packet =
|
||||
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
|
||||
|
||||
// Catch output.
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(
|
||||
graph.AddPacketToInputStream("input_detection", input_detection_packet));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
|
||||
classification_list_packet));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
// Get and validate output.
|
||||
EXPECT_THAT(output_packets, testing::SizeIs(1));
|
||||
const Detection& output_detection = output_packets[0].Get<Detection>();
|
||||
EXPECT_THAT(output_detection,
|
||||
mediapipe::EqualsProto(ParseTextProtoOrDie<Detection>(R"pb(
|
||||
label_id: 11
|
||||
label_id: 12
|
||||
score: 0.5
|
||||
score: 0.4
|
||||
location_data {
|
||||
format: BOUNDING_BOX
|
||||
bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 }
|
||||
}
|
||||
)pb")));
|
||||
}
|
||||
|
||||
// Checks that merging fails if the input ClassificationList misses mandatory
|
||||
// "index" field.
|
||||
TEST(DetectionClassificationsMergerCalculator, FailsWithMissingIndex) {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
|
||||
|
||||
// Prepare input packets.
|
||||
const Detection& input_detection =
|
||||
ParseTextProtoOrDie<Detection>(kInputDetection);
|
||||
Packet input_detection_packet =
|
||||
MakePacket<Detection>(input_detection).At(Timestamp(0));
|
||||
const ClassificationList& classification_list =
|
||||
ParseTextProtoOrDie<ClassificationList>(R"pb(
|
||||
classification { score: 0.5 label: "dog" }
|
||||
)pb");
|
||||
Packet classification_list_packet =
|
||||
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
|
||||
|
||||
// Catch output.
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(
|
||||
graph.AddPacketToInputStream("input_detection", input_detection_packet));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
|
||||
classification_list_packet));
|
||||
ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
|
||||
}
|
||||
|
||||
// Checks that merging fails if the input ClassificationList misses mandatory
|
||||
// "score" field.
|
||||
TEST(DetectionClassificationsMergerCalculator, FailsWithMissingScore) {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
|
||||
|
||||
// Prepare input packets.
|
||||
const Detection& input_detection =
|
||||
ParseTextProtoOrDie<Detection>(kInputDetection);
|
||||
Packet input_detection_packet =
|
||||
MakePacket<Detection>(input_detection).At(Timestamp(0));
|
||||
const ClassificationList& classification_list =
|
||||
ParseTextProtoOrDie<ClassificationList>(R"pb(
|
||||
classification { index: 11 label: "dog" }
|
||||
)pb");
|
||||
Packet classification_list_packet =
|
||||
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
|
||||
|
||||
// Catch output.
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(
|
||||
graph.AddPacketToInputStream("input_detection", input_detection_packet));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
|
||||
classification_list_packet));
|
||||
ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
|
||||
}
|
||||
|
||||
// Checks that merging fails if the input ClassificationList has an
|
||||
// inconsistent number of labels.
|
||||
TEST(DetectionClassificationsMergerCalculator,
|
||||
FailsWithInconsistentNumberOfLabels) {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
|
||||
|
||||
// Prepare input packets.
|
||||
const Detection& input_detection =
|
||||
ParseTextProtoOrDie<Detection>(kInputDetection);
|
||||
Packet input_detection_packet =
|
||||
MakePacket<Detection>(input_detection).At(Timestamp(0));
|
||||
const ClassificationList& classification_list =
|
||||
ParseTextProtoOrDie<ClassificationList>(R"pb(
|
||||
classification { index: 11 score: 0.5 label: "dog" display_name: "Dog" }
|
||||
classification { index: 12 score: 0.4 display_name: "Fox" }
|
||||
)pb");
|
||||
Packet classification_list_packet =
|
||||
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
|
||||
|
||||
// Catch output.
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(
|
||||
graph.AddPacketToInputStream("input_detection", input_detection_packet));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
|
||||
classification_list_packet));
|
||||
ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
|
||||
}
|
||||
|
||||
// Checks that merging fails if the input ClassificationList has an
|
||||
// inconsistent number of display names.
|
||||
TEST(DetectionClassificationsMergerCalculator,
|
||||
FailsWithInconsistentNumberOfDisplayNames) {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
|
||||
|
||||
// Prepare input packets.
|
||||
const Detection& input_detection =
|
||||
ParseTextProtoOrDie<Detection>(kInputDetection);
|
||||
Packet input_detection_packet =
|
||||
MakePacket<Detection>(input_detection).At(Timestamp(0));
|
||||
const ClassificationList& classification_list =
|
||||
ParseTextProtoOrDie<ClassificationList>(R"pb(
|
||||
classification { index: 11 score: 0.5 label: "dog" }
|
||||
classification { index: 12 score: 0.4 label: "fox" display_name: "Fox" }
|
||||
)pb");
|
||||
Packet classification_list_packet =
|
||||
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
|
||||
|
||||
// Catch output.
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(
|
||||
graph.AddPacketToInputStream("input_detection", input_detection_packet));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
|
||||
classification_list_packet));
|
||||
ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -57,12 +57,12 @@ Detection CreateDetection(const std::vector<std::string>& labels,
|
||||
}
|
||||
|
||||
CalculatorGraphConfig::Node GetDefaultNode() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:adjusted_detections"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
TEST(DetectionLetterboxRemovalCalculatorTest, PaddingLeftRight) {
|
||||
|
||||
@@ -66,12 +66,12 @@ std::vector<Point2_f> GetPoints(const Detection& detection) {
|
||||
// Test helper function to run "DetectionProjectionCalculator".
|
||||
absl::StatusOr<Detection> RunProjectionCalculator(
|
||||
Detection detection, std::array<float, 16> project_mat) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionProjectionCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
input_stream: "PROJECTION_MATRIX:matrix"
|
||||
output_stream: "DETECTIONS:projected_detections"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
runner.MutableInputs()
|
||||
->Tag("DETECTIONS")
|
||||
|
||||
@@ -84,11 +84,11 @@ Detection DetectionWithRelativeLocationData(double xmin, double ymin,
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, DetectionToRect) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTION:detection"
|
||||
output_stream: "RECT:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detection = absl::make_unique<Detection>(
|
||||
DetectionWithLocationData(100, 200, 300, 400));
|
||||
@@ -107,7 +107,7 @@ TEST(DetectionsToRectsCalculatorTest, DetectionToRect) {
|
||||
|
||||
absl::StatusOr<Rect> RunDetectionKeyPointsToRectCalculation(
|
||||
Detection detection, std::pair<int, int> image_size) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTION:detection"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
@@ -117,7 +117,7 @@ absl::StatusOr<Rect> RunDetectionKeyPointsToRectCalculation(
|
||||
conversion_mode: USE_KEYPOINTS
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
runner.MutableInputs()
|
||||
->Tag("DETECTION")
|
||||
@@ -160,11 +160,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionKeyPointsToRect) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRect) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTION:detection"
|
||||
output_stream: "NORM_RECT:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detection = absl::make_unique<Detection>(
|
||||
DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
|
||||
@@ -183,7 +183,7 @@ TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRect) {
|
||||
|
||||
absl::StatusOr<NormalizedRect> RunDetectionKeyPointsToNormRectCalculation(
|
||||
Detection detection) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTION:detection"
|
||||
output_stream: "NORM_RECT:rect"
|
||||
@@ -192,7 +192,7 @@ absl::StatusOr<NormalizedRect> RunDetectionKeyPointsToNormRectCalculation(
|
||||
conversion_mode: USE_KEYPOINTS
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
runner.MutableInputs()
|
||||
->Tag("DETECTION")
|
||||
@@ -231,11 +231,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionKeyPointsToNormalizedRect) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, DetectionsToRect) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RECT:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detections(absl::make_unique<std::vector<Detection>>());
|
||||
detections->push_back(DetectionWithLocationData(100, 200, 300, 400));
|
||||
@@ -254,11 +254,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionsToRect) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRect) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "NORM_RECT:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detections(absl::make_unique<std::vector<Detection>>());
|
||||
detections->push_back(DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
|
||||
@@ -277,11 +277,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRect) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, DetectionsToRects) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RECTS:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detections(absl::make_unique<std::vector<Detection>>());
|
||||
detections->push_back(DetectionWithLocationData(100, 200, 300, 400));
|
||||
@@ -302,11 +302,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionsToRects) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRects) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "NORM_RECTS:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detections(absl::make_unique<std::vector<Detection>>());
|
||||
detections->push_back(DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
|
||||
@@ -328,11 +328,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRects) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, DetectionToRects) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTION:detection"
|
||||
output_stream: "RECTS:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detection = absl::make_unique<Detection>(
|
||||
DetectionWithLocationData(100, 200, 300, 400));
|
||||
@@ -351,11 +351,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionToRects) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRects) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTION:detection"
|
||||
output_stream: "NORM_RECTS:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detection = absl::make_unique<Detection>(
|
||||
DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
|
||||
@@ -375,11 +375,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRects) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, WrongInputToRect) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RECT:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detections(absl::make_unique<std::vector<Detection>>());
|
||||
detections->push_back(DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
|
||||
@@ -395,11 +395,11 @@ TEST(DetectionsToRectsCalculatorTest, WrongInputToRect) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRectsCalculatorTest, WrongInputToNormalizedRect) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "NORM_RECT:rect"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto detections(absl::make_unique<std::vector<Detection>>());
|
||||
detections->push_back(DetectionWithLocationData(100, 200, 300, 400));
|
||||
|
||||
@@ -85,11 +85,11 @@ Detection CreateDetection(const std::vector<std::string>& labels,
|
||||
}
|
||||
|
||||
TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionList) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTION_LIST:detection_list"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
LocationData location_data = CreateLocationData(100, 200, 300, 400);
|
||||
auto detections(absl::make_unique<DetectionList>());
|
||||
@@ -119,11 +119,11 @@ TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionList) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionVector) {
|
||||
CalculatorRunner runner{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
)")};
|
||||
)pb")};
|
||||
|
||||
LocationData location_data = CreateLocationData(100, 200, 300, 400);
|
||||
auto detections(absl::make_unique<std::vector<Detection>>());
|
||||
@@ -153,12 +153,12 @@ TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionVector) {
|
||||
}
|
||||
|
||||
TEST(DetectionsToRenderDataCalculatorTest, BothDetecctionListAndVector) {
|
||||
CalculatorRunner runner{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTION_LIST:detection_list"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
)")};
|
||||
)pb")};
|
||||
|
||||
LocationData location_data1 = CreateLocationData(100, 200, 300, 400);
|
||||
auto detection_list(absl::make_unique<DetectionList>());
|
||||
@@ -194,17 +194,18 @@ TEST(DetectionsToRenderDataCalculatorTest, BothDetecctionListAndVector) {
|
||||
|
||||
TEST(DetectionsToRenderDataCalculatorTest, ProduceEmptyPacket) {
|
||||
// Check when produce_empty_packet is false.
|
||||
CalculatorRunner runner1{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTION_LIST:detection_list"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
options {
|
||||
[mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
|
||||
produce_empty_packet: false
|
||||
}
|
||||
}
|
||||
)")};
|
||||
CalculatorRunner runner1{
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTION_LIST:detection_list"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
options {
|
||||
[mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
|
||||
produce_empty_packet: false
|
||||
}
|
||||
}
|
||||
)pb")};
|
||||
|
||||
auto detection_list1(absl::make_unique<DetectionList>());
|
||||
runner1.MutableInputs()
|
||||
@@ -224,17 +225,18 @@ TEST(DetectionsToRenderDataCalculatorTest, ProduceEmptyPacket) {
|
||||
ASSERT_EQ(0, exact1.size());
|
||||
|
||||
// Check when produce_empty_packet is true.
|
||||
CalculatorRunner runner2{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTION_LIST:detection_list"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
options {
|
||||
[mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
|
||||
produce_empty_packet: true
|
||||
}
|
||||
}
|
||||
)")};
|
||||
CalculatorRunner runner2{
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTION_LIST:detection_list"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
options {
|
||||
[mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
|
||||
produce_empty_packet: true
|
||||
}
|
||||
}
|
||||
)pb")};
|
||||
|
||||
auto detection_list2(absl::make_unique<DetectionList>());
|
||||
runner2.MutableInputs()
|
||||
|
||||
@@ -32,12 +32,12 @@ NormalizedLandmark CreateLandmark(float x, float y) {
|
||||
}
|
||||
|
||||
CalculatorGraphConfig::Node GetDefaultNode() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "LandmarkLetterboxRemovalCalculator"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "LANDMARKS:adjusted_landmarks"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingLeftRight) {
|
||||
|
||||
@@ -12,12 +12,15 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/util/filtering/one_euro_filter.h"
|
||||
#include "mediapipe/util/filtering/relative_velocity_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
@@ -25,19 +28,54 @@ namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
|
||||
constexpr char kLandmarksTag[] = "LANDMARKS";
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
|
||||
constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
|
||||
|
||||
using mediapipe::OneEuroFilter;
|
||||
using mediapipe::RelativeVelocityFilter;
|
||||
|
||||
void NormalizedLandmarksToLandmarks(
|
||||
const NormalizedLandmarkList& norm_landmarks, const int image_width,
|
||||
const int image_height, LandmarkList* landmarks) {
|
||||
for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
|
||||
const auto& norm_landmark = norm_landmarks.landmark(i);
|
||||
|
||||
auto* landmark = landmarks->add_landmark();
|
||||
landmark->set_x(norm_landmark.x() * image_width);
|
||||
landmark->set_y(norm_landmark.y() * image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
landmark->set_z(norm_landmark.z() * image_width);
|
||||
landmark->set_visibility(norm_landmark.visibility());
|
||||
landmark->set_presence(norm_landmark.presence());
|
||||
}
|
||||
}
|
||||
|
||||
void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
|
||||
const int image_width,
|
||||
const int image_height,
|
||||
NormalizedLandmarkList* norm_landmarks) {
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const auto& landmark = landmarks.landmark(i);
|
||||
|
||||
auto* norm_landmark = norm_landmarks->add_landmark();
|
||||
norm_landmark->set_x(landmark.x() / image_width);
|
||||
norm_landmark->set_y(landmark.y() / image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
norm_landmark->set_z(landmark.z() / image_width);
|
||||
norm_landmark->set_visibility(landmark.visibility());
|
||||
norm_landmark->set_presence(landmark.presence());
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate object scale to use its inverse value as velocity scale for
|
||||
// RelativeVelocityFilter. If value will be too small (less than
|
||||
// `options_.min_allowed_object_scale`) smoothing will be disabled and
|
||||
// landmarks will be returned as is.
|
||||
// Object scale is calculated as average between bounding box width and height
|
||||
// with sides parallel to axis.
|
||||
float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
|
||||
int image_height) {
|
||||
float GetObjectScale(const LandmarkList& landmarks) {
|
||||
const auto& lm_minmax_x = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.x() < b.x(); });
|
||||
@@ -50,8 +88,8 @@ float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
|
||||
const float y_min = lm_minmax_y.first->y();
|
||||
const float y_max = lm_minmax_y.second->y();
|
||||
|
||||
const float object_width = (x_max - x_min) * image_width;
|
||||
const float object_height = (y_max - y_min) * image_height;
|
||||
const float object_width = x_max - x_min;
|
||||
const float object_height = y_max - y_min;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
@@ -63,19 +101,17 @@ class LandmarksFilter {
|
||||
|
||||
virtual absl::Status Reset() { return absl::OkStatus(); }
|
||||
|
||||
virtual absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
|
||||
const std::pair<int, int>& image_size,
|
||||
virtual absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
NormalizedLandmarkList* out_landmarks) = 0;
|
||||
LandmarkList* out_landmarks) = 0;
|
||||
};
|
||||
|
||||
// Returns landmarks as is without smoothing.
|
||||
class NoFilter : public LandmarksFilter {
|
||||
public:
|
||||
absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
|
||||
const std::pair<int, int>& image_size,
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
NormalizedLandmarkList* out_landmarks) override {
|
||||
LandmarkList* out_landmarks) override {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
@@ -85,10 +121,11 @@ class NoFilter : public LandmarksFilter {
|
||||
class VelocityFilter : public LandmarksFilter {
|
||||
public:
|
||||
VelocityFilter(int window_size, float velocity_scale,
|
||||
float min_allowed_object_scale)
|
||||
float min_allowed_object_scale, bool disable_value_scaling)
|
||||
: window_size_(window_size),
|
||||
velocity_scale_(velocity_scale),
|
||||
min_allowed_object_scale_(min_allowed_object_scale) {}
|
||||
min_allowed_object_scale_(min_allowed_object_scale),
|
||||
disable_value_scaling_(disable_value_scaling) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
@@ -97,45 +134,37 @@ class VelocityFilter : public LandmarksFilter {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
|
||||
const std::pair<int, int>& image_size,
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
NormalizedLandmarkList* out_landmarks) override {
|
||||
// Get image size.
|
||||
int image_width;
|
||||
int image_height;
|
||||
std::tie(image_width, image_height) = image_size;
|
||||
|
||||
LandmarkList* out_landmarks) override {
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
const float object_scale =
|
||||
GetObjectScale(in_landmarks, image_width, image_height);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
float value_scale = 1.0f;
|
||||
if (!disable_value_scaling_) {
|
||||
const float object_scale = GetObjectScale(in_landmarks);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
value_scale = 1.0f / object_scale;
|
||||
}
|
||||
const float value_scale = 1.0f / object_scale;
|
||||
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const NormalizedLandmark& in_landmark = in_landmarks.landmark(i);
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
NormalizedLandmark* out_landmark = out_landmarks->add_landmark();
|
||||
auto* out_landmark = out_landmarks->add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(x_filters_[i].Apply(timestamp, value_scale,
|
||||
in_landmark.x() * image_width) /
|
||||
image_width);
|
||||
out_landmark->set_y(y_filters_[i].Apply(timestamp, value_scale,
|
||||
in_landmark.y() * image_height) /
|
||||
image_height);
|
||||
// Scale Z the save was as X (using image width).
|
||||
out_landmark->set_z(z_filters_[i].Apply(timestamp, value_scale,
|
||||
in_landmark.z() * image_width) /
|
||||
image_width);
|
||||
out_landmark->set_x(
|
||||
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
|
||||
out_landmark->set_y(
|
||||
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
|
||||
out_landmark->set_z(
|
||||
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
@@ -165,12 +194,83 @@ class VelocityFilter : public LandmarksFilter {
|
||||
int window_size_;
|
||||
float velocity_scale_;
|
||||
float min_allowed_object_scale_;
|
||||
bool disable_value_scaling_;
|
||||
|
||||
std::vector<RelativeVelocityFilter> x_filters_;
|
||||
std::vector<RelativeVelocityFilter> y_filters_;
|
||||
std::vector<RelativeVelocityFilter> z_filters_;
|
||||
};
|
||||
|
||||
// Please check OneEuroFilter documentation for details.
|
||||
class OneEuroFilterImpl : public LandmarksFilter {
|
||||
public:
|
||||
OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
|
||||
double derivate_cutoff)
|
||||
: frequency_(frequency),
|
||||
min_cutoff_(min_cutoff),
|
||||
beta_(beta),
|
||||
derivate_cutoff_(derivate_cutoff) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
LandmarkList* out_landmarks) override {
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks->add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(x_filters_[i].Apply(timestamp, in_landmark.x()));
|
||||
out_landmark->set_y(y_filters_[i].Apply(timestamp, in_landmark.y()));
|
||||
out_landmark->set_z(z_filters_[i].Apply(timestamp, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_landmarks; ++i) {
|
||||
x_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
y_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
z_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
double frequency_;
|
||||
double min_cutoff_;
|
||||
double beta_;
|
||||
double derivate_cutoff_;
|
||||
|
||||
std::vector<OneEuroFilter> x_filters_;
|
||||
std::vector<OneEuroFilter> y_filters_;
|
||||
std::vector<OneEuroFilter> z_filters_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
@@ -207,16 +307,21 @@ class LandmarksSmoothingCalculator : public CalculatorBase {
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
LandmarksFilter* landmarks_filter_;
|
||||
std::unique_ptr<LandmarksFilter> landmarks_filter_;
|
||||
};
|
||||
REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
|
||||
|
||||
absl::Status LandmarksSmoothingCalculator::GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Set<NormalizedLandmarkList>();
|
||||
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Set<NormalizedLandmarkList>();
|
||||
} else {
|
||||
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
|
||||
cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
@@ -227,12 +332,19 @@ absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
|
||||
// Pick landmarks filter.
|
||||
const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
|
||||
if (options.has_no_filter()) {
|
||||
landmarks_filter_ = new NoFilter();
|
||||
landmarks_filter_ = absl::make_unique<NoFilter>();
|
||||
} else if (options.has_velocity_filter()) {
|
||||
landmarks_filter_ = new VelocityFilter(
|
||||
landmarks_filter_ = absl::make_unique<VelocityFilter>(
|
||||
options.velocity_filter().window_size(),
|
||||
options.velocity_filter().velocity_scale(),
|
||||
options.velocity_filter().min_allowed_object_scale());
|
||||
options.velocity_filter().min_allowed_object_scale(),
|
||||
options.velocity_filter().disable_value_scaling());
|
||||
} else if (options.has_one_euro_filter()) {
|
||||
landmarks_filter_ = absl::make_unique<OneEuroFilterImpl>(
|
||||
options.one_euro_filter().frequency(),
|
||||
options.one_euro_filter().min_cutoff(),
|
||||
options.one_euro_filter().beta(),
|
||||
options.one_euro_filter().derivate_cutoff());
|
||||
} else {
|
||||
RET_CHECK_FAIL()
|
||||
<< "Landmarks filter is either not specified or not supported";
|
||||
@@ -244,25 +356,53 @@ absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
|
||||
absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
|
||||
if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
|
||||
(cc->Inputs().HasTag(kLandmarksTag) &&
|
||||
cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& in_landmarks =
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
const auto& image_size =
|
||||
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
auto out_landmarks = absl::make_unique<NormalizedLandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(in_landmarks, image_size,
|
||||
timestamp, out_landmarks.get()));
|
||||
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
|
||||
const auto& in_norm_landmarks =
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Add(out_landmarks.release(), cc->InputTimestamp());
|
||||
int image_width;
|
||||
int image_height;
|
||||
std::tie(image_width, image_height) =
|
||||
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
|
||||
auto in_landmarks = absl::make_unique<LandmarkList>();
|
||||
NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width, image_height,
|
||||
in_landmarks.get());
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(*in_landmarks, timestamp,
|
||||
out_landmarks.get()));
|
||||
|
||||
auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
|
||||
LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
|
||||
out_norm_landmarks.get());
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Add(out_norm_landmarks.release(), cc->InputTimestamp());
|
||||
} else {
|
||||
const auto& in_landmarks =
|
||||
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(
|
||||
landmarks_filter_->Apply(in_landmarks, timestamp, out_landmarks.get()));
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kFilteredLandmarksTag)
|
||||
.Add(out_landmarks.release(), cc->InputTimestamp());
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -39,10 +39,40 @@ message LandmarksSmoothingCalculatorOptions {
|
||||
// If calculated object scale is less than given value smoothing will be
|
||||
// disabled and landmarks will be returned as is.
|
||||
optional float min_allowed_object_scale = 3 [default = 1e-6];
|
||||
|
||||
// Disable value scaling based on object size and use `1.0` instead.
|
||||
// Value scale is calculated as inverse value of object size. Object size is
|
||||
// calculated as maximum side of rectangular bounding box of the object in
|
||||
// XY plane.
|
||||
optional bool disable_value_scaling = 4 [default = false];
|
||||
}
|
||||
|
||||
// For the details of the filter implementation and the procedure of its
|
||||
// configuration please check http://cristal.univ-lille.fr/~casiez/1euro/
|
||||
message OneEuroFilter {
|
||||
// Frequency of incomming frames defined in seconds. Used only if can't be
|
||||
// calculated from provided events (e.g. on the very first frame).
|
||||
optional float frequency = 1 [default = 0.033];
|
||||
|
||||
// Minimum cutoff frequency. Start by tuning this parameter while keeping
|
||||
// `beta = 0` to reduce jittering to the desired level. 1Hz (the default
|
||||
// value) is a good starting point.
|
||||
optional float min_cutoff = 2 [default = 1.0];
|
||||
|
||||
// Cutoff slope. After `min_cutoff` is configured, start increasing `beta`
|
||||
// value to reduce the lag introduced by the `min_cutoff`. Find the desired
|
||||
// balance between jittering and lag.
|
||||
optional float beta = 3 [default = 0.0];
|
||||
|
||||
// Cutoff frequency for derivate. It is set to 1Hz in the original
|
||||
// algorithm, but can be tuned to further smooth the speed (i.e. derivate)
|
||||
// on the object.
|
||||
optional float derivate_cutoff = 4 [default = 1.0];
|
||||
}
|
||||
|
||||
oneof filter_options {
|
||||
NoFilter no_filter = 1;
|
||||
VelocityFilter velocity_filter = 2;
|
||||
OneEuroFilter one_euro_filter = 3;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,34 @@ constexpr char kRenderScaleTag[] = "RENDER_SCALE";
|
||||
constexpr char kRenderDataTag[] = "RENDER_DATA";
|
||||
constexpr char kLandmarkLabel[] = "KEYPOINT";
|
||||
|
||||
inline Color DefaultMinDepthLineColor() {
|
||||
Color color;
|
||||
color.set_r(0);
|
||||
color.set_g(0);
|
||||
color.set_b(0);
|
||||
return color;
|
||||
}
|
||||
|
||||
inline Color DefaultMaxDepthLineColor() {
|
||||
Color color;
|
||||
color.set_r(255);
|
||||
color.set_g(255);
|
||||
color.set_b(255);
|
||||
return color;
|
||||
}
|
||||
|
||||
inline Color MixColors(const Color& color1, const Color& color2,
|
||||
float color1_weight) {
|
||||
Color color;
|
||||
color.set_r(static_cast<int>(color1.r() * color1_weight +
|
||||
color2.r() * (1.f - color1_weight)));
|
||||
color.set_g(static_cast<int>(color1.g() * color1_weight +
|
||||
color2.g() * (1.f - color1_weight)));
|
||||
color.set_b(static_cast<int>(color1.b() * color1_weight +
|
||||
color2.b() * (1.f - color1_weight)));
|
||||
return color;
|
||||
}
|
||||
|
||||
inline void SetColor(RenderAnnotation* annotation, const Color& color) {
|
||||
annotation->mutable_color()->set_r(color.r());
|
||||
annotation->mutable_color()->set_g(color.g());
|
||||
@@ -57,6 +85,23 @@ inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
|
||||
}
|
||||
}
|
||||
|
||||
template <class LandmarkType>
|
||||
bool IsLandmarkVisibileAndPresent(const LandmarkType& landmark,
|
||||
bool utilize_visibility,
|
||||
float visibility_threshold,
|
||||
bool utilize_presence,
|
||||
float presence_threshold) {
|
||||
if (utilize_visibility && landmark.has_visibility() &&
|
||||
landmark.visibility() < visibility_threshold) {
|
||||
return false;
|
||||
}
|
||||
if (utilize_presence && landmark.has_presence() &&
|
||||
landmark.presence() < presence_threshold) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void SetColorSizeValueFromZ(float z, float z_min, float z_max,
|
||||
RenderAnnotation* render_annotation,
|
||||
float min_depth_circle_thickness,
|
||||
@@ -75,8 +120,9 @@ void SetColorSizeValueFromZ(float z, float z_min, float z_max,
|
||||
|
||||
template <class LandmarkType>
|
||||
void AddConnectionToRenderData(const LandmarkType& start,
|
||||
const LandmarkType& end, int gray_val1,
|
||||
int gray_val2, float thickness, bool normalized,
|
||||
const LandmarkType& end,
|
||||
const Color& color_start, const Color& color_end,
|
||||
float thickness, bool normalized,
|
||||
RenderData* render_data) {
|
||||
auto* connection_annotation = render_data->add_render_annotations();
|
||||
RenderAnnotation::GradientLine* line =
|
||||
@@ -86,12 +132,13 @@ void AddConnectionToRenderData(const LandmarkType& start,
|
||||
line->set_x_end(end.x());
|
||||
line->set_y_end(end.y());
|
||||
line->set_normalized(normalized);
|
||||
line->mutable_color1()->set_r(gray_val1);
|
||||
line->mutable_color1()->set_g(gray_val1);
|
||||
line->mutable_color1()->set_b(gray_val1);
|
||||
line->mutable_color2()->set_r(gray_val2);
|
||||
line->mutable_color2()->set_g(gray_val2);
|
||||
line->mutable_color2()->set_b(gray_val2);
|
||||
line->mutable_color1()->set_r(color_start.r());
|
||||
line->mutable_color1()->set_g(color_start.g());
|
||||
line->mutable_color1()->set_b(color_start.b());
|
||||
line->mutable_color2()->set_r(color_end.r());
|
||||
line->mutable_color2()->set_g(color_end.g());
|
||||
line->mutable_color2()->set_b(color_end.b());
|
||||
|
||||
connection_annotation->set_thickness(thickness);
|
||||
}
|
||||
|
||||
@@ -102,26 +149,26 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
|
||||
float visibility_threshold, bool utilize_presence,
|
||||
float presence_threshold, float thickness,
|
||||
bool normalized, float min_z, float max_z,
|
||||
const Color& min_depth_line_color,
|
||||
const Color& max_depth_line_color,
|
||||
RenderData* render_data) {
|
||||
for (int i = 0; i < landmark_connections.size(); i += 2) {
|
||||
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
|
||||
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
|
||||
if (utilize_visibility &&
|
||||
((ld0.has_visibility() && ld0.visibility() < visibility_threshold) ||
|
||||
(ld1.has_visibility() && ld1.visibility() < visibility_threshold))) {
|
||||
if (!IsLandmarkVisibileAndPresent<LandmarkType>(
|
||||
ld0, utilize_visibility, visibility_threshold, utilize_presence,
|
||||
presence_threshold) ||
|
||||
!IsLandmarkVisibileAndPresent<LandmarkType>(
|
||||
ld1, utilize_visibility, visibility_threshold, utilize_presence,
|
||||
presence_threshold)) {
|
||||
continue;
|
||||
}
|
||||
if (utilize_presence &&
|
||||
((ld0.has_presence() && ld0.presence() < presence_threshold) ||
|
||||
(ld1.has_presence() && ld1.presence() < presence_threshold))) {
|
||||
continue;
|
||||
}
|
||||
const int gray_val1 =
|
||||
255 - static_cast<int>(Remap(ld0.z(), min_z, max_z, 255));
|
||||
const int gray_val2 =
|
||||
255 - static_cast<int>(Remap(ld1.z(), min_z, max_z, 255));
|
||||
AddConnectionToRenderData<LandmarkType>(ld0, ld1, gray_val1, gray_val2,
|
||||
thickness, normalized, render_data);
|
||||
const Color color0 = MixColors(min_depth_line_color, max_depth_line_color,
|
||||
Remap(ld0.z(), min_z, max_z, 1.f));
|
||||
const Color color1 = MixColors(min_depth_line_color, max_depth_line_color,
|
||||
Remap(ld1.z(), min_z, max_z, 1.f));
|
||||
AddConnectionToRenderData<LandmarkType>(ld0, ld1, color0, color1, thickness,
|
||||
normalized, render_data);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,14 +198,12 @@ void AddConnections(const LandmarkListType& landmarks,
|
||||
for (int i = 0; i < landmark_connections.size(); i += 2) {
|
||||
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
|
||||
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
|
||||
if (utilize_visibility &&
|
||||
((ld0.has_visibility() && ld0.visibility() < visibility_threshold) ||
|
||||
(ld1.has_visibility() && ld1.visibility() < visibility_threshold))) {
|
||||
continue;
|
||||
}
|
||||
if (utilize_presence &&
|
||||
((ld0.has_presence() && ld0.presence() < presence_threshold) ||
|
||||
(ld1.has_presence() && ld1.presence() < presence_threshold))) {
|
||||
if (!IsLandmarkVisibileAndPresent<LandmarkType>(
|
||||
ld0, utilize_visibility, visibility_threshold, utilize_presence,
|
||||
presence_threshold) ||
|
||||
!IsLandmarkVisibileAndPresent<LandmarkType>(
|
||||
ld1, utilize_visibility, visibility_threshold, utilize_presence,
|
||||
presence_threshold)) {
|
||||
continue;
|
||||
}
|
||||
AddConnectionToRenderData<LandmarkType>(ld0, ld1, connection_color,
|
||||
@@ -232,6 +277,13 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
float z_min = 0.f;
|
||||
float z_max = 0.f;
|
||||
|
||||
const Color min_depth_line_color = options_.has_min_depth_line_color()
|
||||
? options_.min_depth_line_color()
|
||||
: DefaultMinDepthLineColor();
|
||||
const Color max_depth_line_color = options_.has_max_depth_line_color()
|
||||
? options_.max_depth_line_color()
|
||||
: DefaultMaxDepthLineColor();
|
||||
|
||||
// Apply scale to `thickness` of rendered landmarks and connections to make
|
||||
// them bigger when object (e.g. pose, hand or face) is closer/bigger and
|
||||
// snaller when object is further/smaller.
|
||||
@@ -254,7 +306,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
landmarks, landmark_connections_, options_.utilize_visibility(),
|
||||
options_.visibility_threshold(), options_.utilize_presence(),
|
||||
options_.presence_threshold(), thickness, /*normalized=*/false, z_min,
|
||||
z_max, render_data.get());
|
||||
z_max, min_depth_line_color, max_depth_line_color, render_data.get());
|
||||
} else {
|
||||
AddConnections<LandmarkList, Landmark>(
|
||||
landmarks, landmark_connections_, options_.utilize_visibility(),
|
||||
@@ -265,13 +317,10 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const Landmark& landmark = landmarks.landmark(i);
|
||||
|
||||
if (options_.utilize_visibility() && landmark.has_visibility() &&
|
||||
landmark.visibility() < options_.visibility_threshold()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (options_.utilize_presence() && landmark.has_presence() &&
|
||||
landmark.presence() < options_.presence_threshold()) {
|
||||
if (!IsLandmarkVisibileAndPresent<Landmark>(
|
||||
landmark, options_.utilize_visibility(),
|
||||
options_.visibility_threshold(), options_.utilize_presence(),
|
||||
options_.presence_threshold())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -303,7 +352,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
landmarks, landmark_connections_, options_.utilize_visibility(),
|
||||
options_.visibility_threshold(), options_.utilize_presence(),
|
||||
options_.presence_threshold(), thickness, /*normalized=*/true, z_min,
|
||||
z_max, render_data.get());
|
||||
z_max, min_depth_line_color, max_depth_line_color, render_data.get());
|
||||
} else {
|
||||
AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
|
||||
landmarks, landmark_connections_, options_.utilize_visibility(),
|
||||
@@ -314,12 +363,10 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const NormalizedLandmark& landmark = landmarks.landmark(i);
|
||||
|
||||
if (options_.utilize_visibility() && landmark.has_visibility() &&
|
||||
landmark.visibility() < options_.visibility_threshold()) {
|
||||
continue;
|
||||
}
|
||||
if (options_.utilize_presence() && landmark.has_presence() &&
|
||||
landmark.presence() < options_.presence_threshold()) {
|
||||
if (!IsLandmarkVisibileAndPresent<NormalizedLandmark>(
|
||||
landmark, options_.utilize_visibility(),
|
||||
options_.visibility_threshold(), options_.utilize_presence(),
|
||||
options_.presence_threshold())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -64,4 +64,10 @@ message LandmarksToRenderDataCalculatorOptions {
|
||||
|
||||
// Max thickness of the drawing for landmark circle.
|
||||
optional double max_depth_circle_thickness = 11 [default = 18.0];
|
||||
|
||||
// Gradient color for the lines connecting landmarks at the minimum depth.
|
||||
optional Color min_depth_line_color = 12;
|
||||
|
||||
// Gradient color for the lines connecting landmarks at the maximum depth.
|
||||
optional Color max_depth_line_color = 13;
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
CalculatorGraphConfig::Node GetDefaultNode() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "PacketFrequencyCalculator"
|
||||
input_stream: "packet_stream"
|
||||
output_stream: "packet_frequency"
|
||||
@@ -34,11 +34,11 @@ CalculatorGraphConfig::Node GetDefaultNode() {
|
||||
label: "stream_description"
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
CalculatorGraphConfig::Node GetNodeWithMultipleStreams() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "PacketFrequencyCalculator"
|
||||
input_stream: "packet_stream_0"
|
||||
input_stream: "packet_stream_1"
|
||||
@@ -55,7 +55,7 @@ CalculatorGraphConfig::Node GetNodeWithMultipleStreams() {
|
||||
label: "stream_description_2"
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
// Tests packet frequency.
|
||||
|
||||
@@ -38,7 +38,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
|
||||
}
|
||||
|
||||
void InitializeSingleStreamGraph() {
|
||||
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "delayed_packet_0"
|
||||
input_stream: "camera_frames"
|
||||
node {
|
||||
@@ -59,7 +59,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
|
||||
input_stream_handler: "ImmediateInputStreamHandler"
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
mediapipe::tool::AddVectorSink("packet_latency_0", &graph_config_,
|
||||
&out_0_packets_);
|
||||
@@ -79,7 +79,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
|
||||
}
|
||||
|
||||
void InitializeMultipleStreamGraph() {
|
||||
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "delayed_packet_0"
|
||||
input_stream: "delayed_packet_1"
|
||||
input_stream: "delayed_packet_2"
|
||||
@@ -107,7 +107,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
|
||||
input_stream_handler: "ImmediateInputStreamHandler"
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
mediapipe::tool::AddVectorSink("packet_latency_0", &graph_config_,
|
||||
&out_0_packets_);
|
||||
@@ -131,7 +131,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
|
||||
}
|
||||
|
||||
void InitializeSingleStreamGraphWithoutClock() {
|
||||
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "delayed_packet_0"
|
||||
input_stream: "camera_frames"
|
||||
node {
|
||||
@@ -150,7 +150,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
|
||||
input_stream_handler: "ImmediateInputStreamHandler"
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
mediapipe::tool::AddVectorSink("packet_latency_0", &graph_config_,
|
||||
&out_0_packets_);
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
namespace mediapipe {
|
||||
|
||||
TEST(TopKScoresCalculatorTest, TestNodeConfig) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TopKScoresCalculator"
|
||||
input_stream: "SCORES:score_vector"
|
||||
output_stream: "TOP_K_INDEXES:top_k_indexes"
|
||||
@@ -32,7 +32,7 @@ TEST(TopKScoresCalculatorTest, TestNodeConfig) {
|
||||
options: {
|
||||
[mediapipe.TopKScoresCalculatorOptions.ext] {}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
auto status = runner.Run();
|
||||
ASSERT_TRUE(!status.ok());
|
||||
@@ -43,7 +43,7 @@ TEST(TopKScoresCalculatorTest, TestNodeConfig) {
|
||||
}
|
||||
|
||||
TEST(TopKScoresCalculatorTest, TestTopKOnly) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TopKScoresCalculator"
|
||||
input_stream: "SCORES:score_vector"
|
||||
output_stream: "TOP_K_INDEXES:top_k_indexes"
|
||||
@@ -51,7 +51,7 @@ TEST(TopKScoresCalculatorTest, TestTopKOnly) {
|
||||
options: {
|
||||
[mediapipe.TopKScoresCalculatorOptions.ext] { top_k: 2 }
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
std::vector<float> score_vector{0.9, 0.2, 0.3, 1.0, 0.1};
|
||||
|
||||
@@ -76,7 +76,7 @@ TEST(TopKScoresCalculatorTest, TestTopKOnly) {
|
||||
}
|
||||
|
||||
TEST(TopKScoresCalculatorTest, TestThresholdOnly) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TopKScoresCalculator"
|
||||
input_stream: "SCORES:score_vector"
|
||||
output_stream: "TOP_K_INDEXES:top_k_indexes"
|
||||
@@ -84,7 +84,7 @@ TEST(TopKScoresCalculatorTest, TestThresholdOnly) {
|
||||
options: {
|
||||
[mediapipe.TopKScoresCalculatorOptions.ext] { threshold: 0.2 }
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
std::vector<float> score_vector{0.9, 0.2, 0.3, 1.0, 0.1};
|
||||
|
||||
@@ -113,7 +113,7 @@ TEST(TopKScoresCalculatorTest, TestThresholdOnly) {
|
||||
}
|
||||
|
||||
TEST(TopKScoresCalculatorTest, TestBothTopKAndThreshold) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TopKScoresCalculator"
|
||||
input_stream: "SCORES:score_vector"
|
||||
output_stream: "TOP_K_INDEXES:top_k_indexes"
|
||||
@@ -121,7 +121,7 @@ TEST(TopKScoresCalculatorTest, TestBothTopKAndThreshold) {
|
||||
options: {
|
||||
[mediapipe.TopKScoresCalculatorOptions.ext] { top_k: 4 threshold: 0.3 }
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
std::vector<float> score_vector{0.9, 0.2, 0.3, 1.0, 0.1};
|
||||
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "mediapipe/calculators/util/visibility_copy_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kLandmarksFromTag[] = "LANDMARKS_FROM";
|
||||
constexpr char kNormalizedLandmarksFromTag[] = "NORM_LANDMARKS_FROM";
|
||||
constexpr char kLandmarksToTag[] = "LANDMARKS_TO";
|
||||
constexpr char kNormalizedLandmarksToTag[] = "NORM_LANDMARKS_TO";
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator to copy visibility and presence between landmarks.
|
||||
//
|
||||
// Landmarks to copy from and to copy to can be of different type (normalized or
|
||||
// non-normalized), but ladnmarks to copy to and output landmarks should be of
|
||||
// the same type. Exactly one stream to copy landmarks from, to copy to and to
|
||||
// output should be provided.
|
||||
//
|
||||
// Inputs:
|
||||
// LANDMARKS_FROM (optional): A LandmarkList of landmarks to copy from.
|
||||
// NORM_LANDMARKS_FROM (optional): A NormalizedLandmarkList of landmarks to
|
||||
// copy from.
|
||||
// LANDMARKS_TO (optional): A LandmarkList of landmarks to copy to.
|
||||
// NORM_LANDMARKS_TO (optional): A NormalizedLandmarkList of landmarks to copy
|
||||
// to.
|
||||
//
|
||||
// Outputs:
|
||||
// LANDMARKS_TO (optional): A LandmarkList of landmarks from LANDMARKS_TO and
|
||||
// visibility/presence from LANDMARKS_FROM or NORM_LANDMARKS_FROM.
|
||||
// NORM_LANDMARKS_TO (optional): A NormalizedLandmarkList of landmarks to copy
|
||||
// to.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "VisibilityCopyCalculator"
|
||||
// input_stream: "NORM_LANDMARKS_FROM:pose_landmarks"
|
||||
// input_stream: "LANDMARKS_TO:pose_world_landmarks"
|
||||
// output_stream: "LANDMARKS_TO:pose_world_landmarks_with_visibility"
|
||||
// options: {
|
||||
// [mediapipe.VisibilityCopyCalculatorOptions.ext] {
|
||||
// copy_visibility: true
|
||||
// copy_presence: true
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class VisibilityCopyCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
template <class LandmarkFromType, class LandmarkToType>
|
||||
absl::Status CopyVisibility(CalculatorContext* cc,
|
||||
const std::string& landmarks_from_tag,
|
||||
const std::string& landmarks_to_tag);
|
||||
|
||||
bool copy_visibility_;
|
||||
bool copy_presence_;
|
||||
};
|
||||
REGISTER_CALCULATOR(VisibilityCopyCalculator);
|
||||
|
||||
absl::Status VisibilityCopyCalculator::GetContract(CalculatorContract* cc) {
|
||||
// Landmarks to copy from.
|
||||
RET_CHECK(cc->Inputs().HasTag(kLandmarksFromTag) ^
|
||||
cc->Inputs().HasTag(kNormalizedLandmarksFromTag))
|
||||
<< "Exatly one landmarks stream to copy from should be provided";
|
||||
if (cc->Inputs().HasTag(kLandmarksFromTag)) {
|
||||
cc->Inputs().Tag(kLandmarksFromTag).Set<LandmarkList>();
|
||||
} else {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksFromTag).Set<NormalizedLandmarkList>();
|
||||
}
|
||||
|
||||
// Landmarks to copy to and corresponding output landmarks.
|
||||
RET_CHECK(cc->Inputs().HasTag(kLandmarksToTag) ^
|
||||
cc->Inputs().HasTag(kNormalizedLandmarksToTag))
|
||||
<< "Exatly one landmarks stream to copy to should be provided";
|
||||
if (cc->Inputs().HasTag(kLandmarksToTag)) {
|
||||
cc->Inputs().Tag(kLandmarksToTag).Set<LandmarkList>();
|
||||
|
||||
RET_CHECK(cc->Outputs().HasTag(kLandmarksToTag))
|
||||
<< "Landmarks to copy to and output stream types should be the same";
|
||||
cc->Outputs().Tag(kLandmarksToTag).Set<LandmarkList>();
|
||||
} else {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksToTag).Set<NormalizedLandmarkList>();
|
||||
|
||||
RET_CHECK(cc->Outputs().HasTag(kNormalizedLandmarksToTag))
|
||||
<< "Landmarks to copy to and output stream types should be the same";
|
||||
cc->Outputs().Tag(kNormalizedLandmarksToTag).Set<NormalizedLandmarkList>();
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status VisibilityCopyCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
const auto& options = cc->Options<VisibilityCopyCalculatorOptions>();
|
||||
copy_visibility_ = options.copy_visibility();
|
||||
copy_presence_ = options.copy_presence();
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status VisibilityCopyCalculator::Process(CalculatorContext* cc) {
|
||||
// Switch between all four possible combinations of landmarks from and
|
||||
// landmarks to types (normalized and non-normalized).
|
||||
auto status = absl::OkStatus();
|
||||
if (cc->Inputs().HasTag(kLandmarksFromTag)) {
|
||||
if (cc->Inputs().HasTag(kLandmarksToTag)) {
|
||||
status = CopyVisibility<LandmarkList, LandmarkList>(cc, kLandmarksFromTag,
|
||||
kLandmarksToTag);
|
||||
} else {
|
||||
status = CopyVisibility<LandmarkList, NormalizedLandmarkList>(
|
||||
cc, kLandmarksFromTag, kNormalizedLandmarksToTag);
|
||||
}
|
||||
} else {
|
||||
if (cc->Inputs().HasTag(kLandmarksToTag)) {
|
||||
status = CopyVisibility<NormalizedLandmarkList, LandmarkList>(
|
||||
cc, kNormalizedLandmarksFromTag, kLandmarksToTag);
|
||||
} else {
|
||||
status = CopyVisibility<NormalizedLandmarkList, NormalizedLandmarkList>(
|
||||
cc, kNormalizedLandmarksFromTag, kNormalizedLandmarksToTag);
|
||||
}
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
template <class LandmarkFromType, class LandmarkToType>
|
||||
absl::Status VisibilityCopyCalculator::CopyVisibility(
|
||||
CalculatorContext* cc, const std::string& landmarks_from_tag,
|
||||
const std::string& landmarks_to_tag) {
|
||||
// Check that both landmarks to copy from and to copy to are non empty.
|
||||
if (cc->Inputs().Tag(landmarks_from_tag).IsEmpty() ||
|
||||
cc->Inputs().Tag(landmarks_to_tag).IsEmpty()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto landmarks_from =
|
||||
cc->Inputs().Tag(landmarks_from_tag).Get<LandmarkFromType>();
|
||||
const auto landmarks_to =
|
||||
cc->Inputs().Tag(landmarks_to_tag).Get<LandmarkToType>();
|
||||
auto landmarks_out = absl::make_unique<LandmarkToType>();
|
||||
|
||||
for (int i = 0; i < landmarks_from.landmark_size(); ++i) {
|
||||
const auto& landmark_from = landmarks_from.landmark(i);
|
||||
const auto& landmark_to = landmarks_to.landmark(i);
|
||||
|
||||
// Create output landmark and copy all fields from the `to` landmark.
|
||||
const auto& landmark_out = landmarks_out->add_landmark();
|
||||
*landmark_out = landmark_to;
|
||||
|
||||
// Copy visibility and presence from the `from` landmark.
|
||||
if (copy_visibility_) {
|
||||
landmark_out->set_visibility(landmark_from.visibility());
|
||||
}
|
||||
if (copy_presence_) {
|
||||
landmark_out->set_presence(landmark_from.presence());
|
||||
}
|
||||
}
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(landmarks_to_tag)
|
||||
.Add(landmarks_out.release(), cc->InputTimestamp());
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,29 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator_options.proto";
|
||||
|
||||
message VisibilityCopyCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional VisibilityCopyCalculatorOptions ext = 363728421;
|
||||
}
|
||||
|
||||
optional bool copy_visibility = 1 [default = true];
|
||||
|
||||
optional bool copy_presence = 2 [default = true];
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "mediapipe/calculators/util/visibility_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/util/filtering/low_pass_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
|
||||
constexpr char kLandmarksTag[] = "LANDMARKS";
|
||||
constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
|
||||
constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
|
||||
|
||||
using mediapipe::LowPassFilter;
|
||||
|
||||
// Abstract class for various visibility filters.
|
||||
class VisibilityFilter {
|
||||
public:
|
||||
virtual ~VisibilityFilter() = default;
|
||||
|
||||
virtual absl::Status Reset() { return absl::OkStatus(); }
|
||||
|
||||
virtual absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
LandmarkList* out_landmarks) = 0;
|
||||
|
||||
virtual absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
NormalizedLandmarkList* out_landmarks) = 0;
|
||||
};
|
||||
|
||||
// Returns visibility as is without smoothing.
|
||||
class NoFilter : public VisibilityFilter {
|
||||
public:
|
||||
absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
NormalizedLandmarkList* out_landmarks) override {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
LandmarkList* out_landmarks) override {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
// Please check LowPassFilter documentation for details.
|
||||
class LowPassVisibilityFilter : public VisibilityFilter {
|
||||
public:
|
||||
LowPassVisibilityFilter(float alpha) : alpha_(alpha) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
visibility_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
LandmarkList* out_landmarks) override {
|
||||
return ApplyImpl<LandmarkList>(in_landmarks, timestamp, out_landmarks);
|
||||
}
|
||||
|
||||
absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
NormalizedLandmarkList* out_landmarks) override {
|
||||
return ApplyImpl<NormalizedLandmarkList>(in_landmarks, timestamp,
|
||||
out_landmarks);
|
||||
}
|
||||
|
||||
private:
|
||||
template <class LandmarksType>
|
||||
absl::Status ApplyImpl(const LandmarksType& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
LandmarksType* out_landmarks) {
|
||||
// Initializes filters for the first time or after Reset. If initialized
|
||||
// then check the size.
|
||||
int n_landmarks = in_landmarks.landmark_size();
|
||||
if (!visibility_filters_.empty()) {
|
||||
RET_CHECK_EQ(visibility_filters_.size(), n_landmarks);
|
||||
} else {
|
||||
visibility_filters_.resize(n_landmarks, LowPassFilter(alpha_));
|
||||
}
|
||||
|
||||
// Filter visibilities.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks->add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_visibility(
|
||||
visibility_filters_[i].Apply(in_landmark.visibility()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
float alpha_;
|
||||
std::vector<LowPassFilter> visibility_filters_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator to smooth landmark visibilities over time.
|
||||
//
|
||||
// Exactly one landmarks input stream is expected. Output stream type should be
|
||||
// the same as the input one.
|
||||
//
|
||||
// Inputs:
|
||||
// LANDMARKS (optional): A LandmarkList of landmarks you want to smooth.
|
||||
// NORM_LANDMARKS (optional): A NormalizedLandmarkList of landmarks you want
|
||||
// to smooth.
|
||||
//
|
||||
// Outputs:
|
||||
// FILTERED_LANDMARKS (optional): A LandmarkList of smoothed landmarks.
|
||||
// NORM_FILTERED_LANDMARKS (optional): A NormalizedLandmarkList of smoothed
|
||||
// landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "VisibilitySmoothingCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:pose_landmarks"
|
||||
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
|
||||
// options: {
|
||||
// [mediapipe.VisibilitySmoothingCalculatorOptions.ext] {
|
||||
// low_pass_filter: {
|
||||
// alpha: 0.1
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class VisibilitySmoothingCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
std::unique_ptr<VisibilityFilter> visibility_filter_;
|
||||
};
|
||||
REGISTER_CALCULATOR(VisibilitySmoothingCalculator);
|
||||
|
||||
absl::Status VisibilitySmoothingCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kNormalizedLandmarksTag) ^
|
||||
cc->Inputs().HasTag(kLandmarksTag))
|
||||
<< "Exactly one landmarks input stream is expected";
|
||||
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
RET_CHECK(cc->Outputs().HasTag(kNormalizedFilteredLandmarksTag))
|
||||
<< "Landmarks output stream should of the same type as input one";
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Set<NormalizedLandmarkList>();
|
||||
} else {
|
||||
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
|
||||
RET_CHECK(cc->Outputs().HasTag(kFilteredLandmarksTag))
|
||||
<< "Landmarks output stream should of the same type as input one";
|
||||
cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status VisibilitySmoothingCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
// Pick visibility filter.
|
||||
const auto& options = cc->Options<VisibilitySmoothingCalculatorOptions>();
|
||||
if (options.has_no_filter()) {
|
||||
visibility_filter_ = absl::make_unique<NoFilter>();
|
||||
} else if (options.has_low_pass_filter()) {
|
||||
visibility_filter_ = absl::make_unique<LowPassVisibilityFilter>(
|
||||
options.low_pass_filter().alpha());
|
||||
} else {
|
||||
RET_CHECK_FAIL()
|
||||
<< "Visibility filter is either not specified or not supported";
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status VisibilitySmoothingCalculator::Process(CalculatorContext* cc) {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
|
||||
(cc->Inputs().HasTag(kLandmarksTag) &&
|
||||
cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
|
||||
MP_RETURN_IF_ERROR(visibility_filter_->Reset());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
|
||||
const auto& in_landmarks =
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
auto out_landmarks = absl::make_unique<NormalizedLandmarkList>();
|
||||
MP_RETURN_IF_ERROR(visibility_filter_->Apply(in_landmarks, timestamp,
|
||||
out_landmarks.get()));
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Add(out_landmarks.release(), cc->InputTimestamp());
|
||||
} else {
|
||||
const auto& in_landmarks =
|
||||
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(visibility_filter_->Apply(in_landmarks, timestamp,
|
||||
out_landmarks.get()));
|
||||
cc->Outputs()
|
||||
.Tag(kFilteredLandmarksTag)
|
||||
.Add(out_landmarks.release(), cc->InputTimestamp());
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator_options.proto";
|
||||
|
||||
message VisibilitySmoothingCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional VisibilitySmoothingCalculatorOptions ext = 360207350;
|
||||
}
|
||||
|
||||
// Default behaviour and fast way to disable smoothing.
|
||||
message NoFilter {}
|
||||
|
||||
message LowPassFilter {
|
||||
// Coefficient applied to a new value, whilte `1 - alpha` is applied to a
|
||||
// stored value. Should be in [0, 1] range. The smaller the value - the
|
||||
// smoother result and the bigger lag.
|
||||
optional float alpha = 1 [default = 0.1];
|
||||
}
|
||||
|
||||
oneof filter_options {
|
||||
NoFilter no_filter = 1;
|
||||
LowPassFilter low_pass_filter = 2;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kLandmarksTag[] = "LANDMARKS";
|
||||
constexpr char kRectTag[] = "NORM_RECT";
|
||||
|
||||
} // namespace
|
||||
|
||||
// Projects world landmarks from the rectangle to original coordinates.
|
||||
//
|
||||
// World landmarks are predicted in meters rather than in pixels of the image
|
||||
// and have origin in the middle of the hips rather than in the corner of the
|
||||
// pose image (cropped with given rectangle). Thus only rotation (but not scale
|
||||
// and translation) is applied to the landmarks to transform them back to
|
||||
// original coordinates.
|
||||
//
|
||||
// Input:
|
||||
// LANDMARKS: A LandmarkList representing world landmarks in the rectangle.
|
||||
// NORM_RECT: An NormalizedRect representing a normalized rectangle in image
|
||||
// coordinates.
|
||||
//
|
||||
// Output:
|
||||
// LANDMARKS: A LandmarkList representing world landmarks projected (rotated
|
||||
// but not scaled or translated) from the rectangle to original
|
||||
// coordinates.
|
||||
//
|
||||
// Usage example:
|
||||
// node {
|
||||
// calculator: "WorldLandmarkProjectionCalculator"
|
||||
// input_stream: "LANDMARKS:landmarks"
|
||||
// input_stream: "NORM_RECT:rect"
|
||||
// output_stream: "LANDMARKS:projected_landmarks"
|
||||
// }
|
||||
//
|
||||
class WorldLandmarkProjectionCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
|
||||
cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
|
||||
cc->Outputs().Tag(kLandmarksTag).Set<LandmarkList>();
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
// Check that landmarks and rect are not empty.
|
||||
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty() ||
|
||||
cc->Inputs().Tag(kRectTag).IsEmpty()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& in_landmarks =
|
||||
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
|
||||
const auto& in_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
Landmark* out_landmark = out_landmarks->add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
|
||||
const float angle = in_rect.rotation();
|
||||
out_landmark->set_x(std::cos(angle) * in_landmark.x() -
|
||||
std::sin(angle) * in_landmark.y());
|
||||
out_landmark->set_y(std::sin(angle) * in_landmark.x() +
|
||||
std::cos(angle) * in_landmark.y());
|
||||
}
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kLandmarksTag)
|
||||
.Add(out_landmarks.release(), cc->InputTimestamp());
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(WorldLandmarkProjectionCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -426,6 +426,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -450,6 +451,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:opencv_video",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -534,6 +536,7 @@ cc_test(
|
||||
"//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
|
||||
"//mediapipe/util/tracking:box_tracker_cc_proto",
|
||||
"//mediapipe/util/tracking:tracking_cc_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -31,11 +31,11 @@ namespace {
|
||||
|
||||
TEST(OpenCvVideoDecoderCalculatorTest, TestMp4Avc720pVideo) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "VIDEO:video"
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream")");
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream")pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -77,11 +77,11 @@ TEST(OpenCvVideoDecoderCalculatorTest, TestMp4Avc720pVideo) {
|
||||
|
||||
TEST(OpenCvVideoDecoderCalculatorTest, TestFlvH264Video) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "VIDEO:video"
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream")");
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream")pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -121,11 +121,11 @@ TEST(OpenCvVideoDecoderCalculatorTest, TestFlvH264Video) {
|
||||
|
||||
TEST(OpenCvVideoDecoderCalculatorTest, TestMkvVp8Video) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "VIDEO:video"
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream")");
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream")pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
|
||||
@@ -36,26 +36,28 @@ namespace {
|
||||
// TODO: Investigate the “Could not open codec 'libx264'” error with
|
||||
// opencv2.
|
||||
TEST(OpenCvVideoEncoderCalculatorTest, DISABLED_TestMp4Avc720pVideo) {
|
||||
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "VIDEO:video"
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
}
|
||||
node {
|
||||
calculator: "OpenCvVideoEncoderCalculator"
|
||||
input_stream: "VIDEO:video"
|
||||
input_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
|
||||
node_options {
|
||||
[type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
|
||||
codec: "avc1"
|
||||
video_format: "mp4"
|
||||
CalculatorGraphConfig config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "VIDEO:video"
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
node {
|
||||
calculator: "OpenCvVideoEncoderCalculator"
|
||||
input_stream: "VIDEO:video"
|
||||
input_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
|
||||
node_options {
|
||||
[type.googleapis.com/
|
||||
mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
|
||||
codec: "avc1"
|
||||
video_format: "mp4"
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
std::map<std::string, Packet> input_side_packets;
|
||||
input_side_packets["input_file_path"] = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -95,26 +97,28 @@ TEST(OpenCvVideoEncoderCalculatorTest, DISABLED_TestMp4Avc720pVideo) {
|
||||
}
|
||||
|
||||
TEST(OpenCvVideoEncoderCalculatorTest, TestFlvH264Video) {
|
||||
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "VIDEO:video"
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
}
|
||||
node {
|
||||
calculator: "OpenCvVideoEncoderCalculator"
|
||||
input_stream: "VIDEO:video"
|
||||
input_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
|
||||
node_options {
|
||||
[type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
|
||||
codec: "MJPG"
|
||||
video_format: "avi"
|
||||
CalculatorGraphConfig config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "VIDEO:video"
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
node {
|
||||
calculator: "OpenCvVideoEncoderCalculator"
|
||||
input_stream: "VIDEO:video"
|
||||
input_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
|
||||
node_options {
|
||||
[type.googleapis.com/
|
||||
mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
|
||||
codec: "MJPG"
|
||||
video_format: "avi"
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
std::map<std::string, Packet> input_side_packets;
|
||||
input_side_packets["input_file_path"] = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -156,26 +160,28 @@ TEST(OpenCvVideoEncoderCalculatorTest, TestFlvH264Video) {
|
||||
}
|
||||
|
||||
TEST(OpenCvVideoEncoderCalculatorTest, TestMkvVp8Video) {
|
||||
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "VIDEO:video"
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
}
|
||||
node {
|
||||
calculator: "OpenCvVideoEncoderCalculator"
|
||||
input_stream: "VIDEO:video"
|
||||
input_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
|
||||
node_options {
|
||||
[type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
|
||||
codec: "PIM1"
|
||||
video_format: "mkv"
|
||||
CalculatorGraphConfig config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "VIDEO:video"
|
||||
output_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
node {
|
||||
calculator: "OpenCvVideoEncoderCalculator"
|
||||
input_stream: "VIDEO:video"
|
||||
input_stream: "VIDEO_PRESTREAM:video_prestream"
|
||||
input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
|
||||
node_options {
|
||||
[type.googleapis.com/
|
||||
mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
|
||||
codec: "PIM1"
|
||||
video_format: "mkv"
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
std::map<std::string, Packet> input_side_packets;
|
||||
input_side_packets["input_file_path"] = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user