Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de4fbc10e6 | ||
|
|
d144e564d8 | ||
|
|
dd02df1dbe | ||
|
|
66b377c825 | ||
|
|
bf5185f122 | ||
|
|
a2823541e6 | ||
|
|
ae6be10afe | ||
|
|
38ee2603a7 | ||
|
|
86b3283b2f | ||
|
|
7d470a1335 | ||
|
|
d16cc3be5b | ||
|
|
137867d088 | ||
|
|
446d7cf6b6 | ||
|
|
90f72bd851 | ||
|
|
4285aeddfc | ||
|
|
37287925b0 | ||
|
|
48bcbb115f | ||
|
|
9437483827 | ||
|
|
d030c13931 | ||
|
|
fce372d153 | ||
|
|
c6fea4c9d9 | ||
|
|
259b48e082 | ||
|
|
a2a63e3876 | ||
|
|
412ab42d1f | ||
|
|
cc1a02c54f | ||
|
|
b27c562e45 | ||
|
|
61bc4556af | ||
|
|
423c21b454 | ||
|
|
785d266e3f | ||
|
|
59a398924f | ||
|
|
dc9216dc59 | ||
|
|
af67642055 | ||
|
|
731d2b9536 | ||
|
|
b83cfcc9b5 | ||
|
|
9d45360bc9 | ||
|
|
f5df228d9b | ||
|
|
71a47bb18b | ||
|
|
c27a7c1e10 | ||
|
|
7d4ec1e24d | ||
|
|
294687295d |
@@ -3,7 +3,7 @@
|
||||
# Basic build settings
|
||||
build --jobs 128
|
||||
build --define='absl=1'
|
||||
build --cxxopt='-std=c++11'
|
||||
build --cxxopt='-std=c++14'
|
||||
build --copt='-Wno-sign-compare'
|
||||
build --copt='-Wno-unused-function'
|
||||
build --copt='-Wno-uninitialized'
|
||||
@@ -12,12 +12,16 @@ build --copt='-Wno-comment'
|
||||
build --copt='-Wno-return-type'
|
||||
build --copt='-Wno-unused-local-typedefs'
|
||||
build --copt='-Wno-ignored-attributes'
|
||||
# Temporarily set the incompatiblity flag for Bazel 0.27.0 and above
|
||||
# Temporarily set the incompatibility flag for Bazel 0.27.0 and above
|
||||
build --incompatible_disable_deprecated_attr_params=false
|
||||
build --incompatible_depset_is_not_iterable=false
|
||||
|
||||
# Sets the default Apple platform to macOS.
|
||||
build --apple_platform_type=macos
|
||||
|
||||
# Allow debugging with XCODE
|
||||
build --apple_generate_dsym
|
||||
|
||||
# Android configs.
|
||||
build:android --crosstool_top=//external:android/crosstool
|
||||
build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
|
||||
@@ -34,3 +38,30 @@ build:android_arm --fat_apk_cpu=armeabi-v7a
|
||||
build:android_arm64 --config=android
|
||||
build:android_arm64 --cpu=arm64-v8a
|
||||
build:android_arm64 --fat_apk_cpu=arm64-v8a
|
||||
|
||||
# iOS configs.
|
||||
build:ios --apple_platform_type=ios
|
||||
|
||||
build:ios_i386 --config=ios
|
||||
build:ios_i386 --cpu=ios_i386
|
||||
build:ios_i386 --watchos_cpus=i386
|
||||
|
||||
build:ios_x86_64 --config=ios
|
||||
build:ios_x86_64 --cpu=ios_x86_64
|
||||
build:ios_x86_64 --watchos_cpus=i386
|
||||
|
||||
build:ios_armv7 --config=ios
|
||||
build:ios_armv7 --cpu=ios_armv7
|
||||
build:ios_armv7 --watchos_cpus=armv7k
|
||||
|
||||
build:ios_arm64 --config=ios
|
||||
build:ios_arm64 --cpu=ios_arm64
|
||||
build:ios_arm64 --watchos_cpus=armv7k
|
||||
|
||||
build:ios_arm64e --config=ios
|
||||
build:ios_arm64e --cpu=ios_arm64e
|
||||
build:ios_arm64e --watchos_cpus=armv7k
|
||||
|
||||
build:ios_fat --config=ios
|
||||
build:ios_fat --ios_multi_cpus=armv7,arm64
|
||||
build:ios_fat --watchos_cpus=armv7k
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
mediapipe/provisioning_profile.mobileprovision
|
||||
bazel-bin
|
||||
bazel-genfiles
|
||||
bazel-mediapipe-ioss
|
||||
bazel-out
|
||||
bazel-testlogs
|
||||
mediapipe/MediaPipe.xcodeproj
|
||||
mediapipe/MediaPipe.tulsiproj/*.tulsiconf-user
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright 2019 The MediaPipeOSS Authors.
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
+14
-3
@@ -24,20 +24,31 @@ ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
wget \
|
||||
unzip \
|
||||
python \
|
||||
python-pip \
|
||||
python3-pip \
|
||||
libopencv-core-dev \
|
||||
libopencv-highgui-dev \
|
||||
libopencv-imgproc-dev \
|
||||
libopencv-video-dev \
|
||||
&& \
|
||||
libopencv-calib3d-dev \
|
||||
libopencv-features2d-dev \
|
||||
software-properties-common && \
|
||||
add-apt-repository -y ppa:openjdk-r/ppa && \
|
||||
apt-get update && apt-get install -y openjdk-8-jdk && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip install --upgrade setuptools
|
||||
RUN pip install future
|
||||
RUN pip3 install six
|
||||
|
||||
# Install bazel
|
||||
ARG BAZEL_VERSION=0.26.1
|
||||
ARG BAZEL_VERSION=1.1.0
|
||||
RUN mkdir /bazel && \
|
||||
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
|
||||
azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||
@@ -49,4 +60,4 @@ azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||
COPY . /mediapipe/
|
||||
|
||||
# If we want the docker image to contain the pre-built object_detection_offline_demo binary, do the following
|
||||
# RUN bazel build -c opt --define 'MEDIAPIPE_DISABLE_GPU=1' mediapipe/examples/desktop/demo:object_detection_tensorflow_demo
|
||||
# RUN bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/demo:object_detection_tensorflow_demo
|
||||
|
||||
@@ -1,15 +1,35 @@
|
||||

|
||||
=======================================================================
|
||||
|
||||
[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.
|
||||
[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data), cross platform (i.e Android, iOS, web, edge devices) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.
|
||||
|
||||

|
||||
|
||||
> "<em>MediaPipe has made it extremely easy to build our 3D person pose reconstruction demo app, facilitating accelerated neural network inference on device and synchronization of our result visualization with the video capture stream. Highly recommended!</em>" - George Papandreou, CTO, [Ariel AI](https://arielai.com)
|
||||
|
||||
## ML Solutions in MediaPipe
|
||||
|
||||
* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
|
||||
* [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
|
||||
* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
|
||||
* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
|
||||
* [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
|
||||
* [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
|
||||
* [AutoFlip](mediapipe/docs/autoflip.md)
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||

|
||||
|
||||
## Installation
|
||||
Follow these [instructions](mediapipe/docs/install.md).
|
||||
|
||||
## Getting started
|
||||
See mobile and desktop [examples](mediapipe/docs/examples.md).
|
||||
See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).
|
||||
|
||||
Check out some web demos [[Edge detection]](https://viz.mediapipe.dev/runner/demos/edge_detection/edge_detection.html) [[Face detection]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html) [[Hand Tracking]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
|
||||
|
||||
## Documentation
|
||||
[MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev)
|
||||
@@ -19,17 +39,31 @@ Check out the [Examples page](https://mediapipe.readthedocs.io/en/latest/example
|
||||
## Visualizing MediaPipe graphs
|
||||
A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.dev/). Please also see instructions [here](mediapipe/docs/visualizer.md).
|
||||
|
||||
## Community forum
|
||||
* [discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
|
||||
## Videos
|
||||
* [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)
|
||||
|
||||
## Publications
|
||||
* [Google Developer Blog: MediaPipe on the Web](https://mediapipe.page.link/webdevblog)
|
||||
* [Google Developer Blog: Object Detection and Tracking using MediaPipe](https://mediapipe.page.link/objecttrackingblog)
|
||||
* [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
|
||||
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
||||
|
||||
## Events
|
||||
[Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
|
||||
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
|
||||
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
|
||||
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
|
||||
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
|
||||
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
|
||||
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop](https://research.google.com/youtube8m/workshop2019/index.html) Seoul, Korea ICCV 2019
|
||||
* [AI DevWorld 2019](https://aidevworld.com) on Oct 10 in San Jose, California
|
||||
* [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan
|
||||
* [Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
|
||||
|
||||
## Community forum
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
|
||||
|
||||
## Alpha Disclaimer
|
||||
MediaPipe is currently in alpha for v0.5. We are still making breaking API changes and expect to get to stable API by v1.0.
|
||||
MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.
|
||||
|
||||
## Contributing
|
||||
We welcome contributions. Please follow these [guidelines](./CONTRIBUTING.md).
|
||||
|
||||
@@ -2,26 +2,39 @@ workspace(name = "mediapipe")
|
||||
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
skylib_version = "0.8.0"
|
||||
http_archive(
|
||||
name = "bazel_skylib",
|
||||
sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d",
|
||||
strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b",
|
||||
urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"],
|
||||
type = "tar.gz",
|
||||
url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version),
|
||||
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
|
||||
)
|
||||
load("@bazel_skylib//lib:versions.bzl", "versions")
|
||||
versions.check(minimum_bazel_version = "0.23.0")
|
||||
versions.check(minimum_bazel_version = "0.24.1",
|
||||
maximum_bazel_version = "1.2.1")
|
||||
|
||||
# ABSL cpp library.
|
||||
|
||||
# ABSL cpp library lts_2019_08_08.
|
||||
http_archive(
|
||||
name = "com_google_absl",
|
||||
# Head commit on 2019-04-12.
|
||||
# TODO: Switch to the latest absl version when the problem gets
|
||||
# fixed.
|
||||
urls = [
|
||||
"https://github.com/abseil/abseil-cpp/archive/a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a.tar.gz",
|
||||
"https://github.com/abseil/abseil-cpp/archive/20190808.tar.gz",
|
||||
],
|
||||
sha256 = "d437920d1434c766d22e85773b899c77c672b8b4865d5dc2cd61a29fdff3cf03",
|
||||
strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
|
||||
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
|
||||
patches = [
|
||||
"@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "abseil-cpp-20190808",
|
||||
sha256 = "8100085dada279bf3ee00cd064d43b5f55e5d913be0dfe2906f06f8f28d5b37e"
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "rules_cc",
|
||||
strip_prefix = "rules_cc-master",
|
||||
urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
|
||||
)
|
||||
|
||||
# GoogleTest/GoogleMock framework. Used by most unit-tests.
|
||||
@@ -52,11 +65,25 @@ http_archive(
|
||||
|
||||
# glog
|
||||
http_archive(
|
||||
name = "com_google_glog",
|
||||
name = "com_github_glog_glog",
|
||||
url = "https://github.com/google/glog/archive/v0.3.5.zip",
|
||||
sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
|
||||
strip_prefix = "glog-0.3.5",
|
||||
build_file = "@//third_party:glog.BUILD",
|
||||
patches = [
|
||||
"@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
)
|
||||
|
||||
# easyexif
|
||||
http_archive(
|
||||
name = "easyexif",
|
||||
url = "https://github.com/mayanklahiri/easyexif/archive/master.zip",
|
||||
strip_prefix = "easyexif-master",
|
||||
build_file = "@//third_party:easyexif.BUILD",
|
||||
)
|
||||
|
||||
# libyuv
|
||||
@@ -73,6 +100,12 @@ http_archive(
|
||||
urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"],
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "com_google_audio_tools",
|
||||
strip_prefix = "multichannel-audio-tools-master",
|
||||
urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
|
||||
)
|
||||
|
||||
# Needed by TensorFlow
|
||||
http_archive(
|
||||
name = "io_bazel_rules_closure",
|
||||
@@ -84,38 +117,89 @@ http_archive(
|
||||
],
|
||||
)
|
||||
|
||||
# TensorFlow r1.14-rc0
|
||||
# 2019-11-21
|
||||
_TENSORFLOW_GIT_COMMIT = "f482488b481a799ca07e7e2d153cf47b8e91a60c"
|
||||
_TENSORFLOW_SHA256= "8d9118c2ce186c7e1403f04b96982fe72c184060c7f7a93e30a28dca358694f0"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
strip_prefix = "tensorflow-1.14.0-rc0",
|
||||
sha256 = "76404a6157a45e8d7a07e4f5690275256260130145924c2a7c73f6eda2a3de10",
|
||||
urls = ["https://github.com/tensorflow/tensorflow/archive/v1.14.0-rc0.zip"],
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
],
|
||||
# Patch https://github.com/tensorflow/tensorflow/commit/e3a7bdbebb99352351a19e2e403136166aa52934
|
||||
patches = [
|
||||
"@//third_party:org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
|
||||
sha256 = _TENSORFLOW_SHA256,
|
||||
)
|
||||
|
||||
load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
|
||||
tf_workspace(tf_repo_name = "org_tensorflow")
|
||||
|
||||
# Please run $ sudo apt-get install libopencv-dev
|
||||
http_archive(
|
||||
name = "ceres_solver",
|
||||
url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
|
||||
patches = [
|
||||
"@//third_party:ceres_solver_9bf9588988236279e1262f75d7f4d85711dfa172.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "ceres-solver-1.14.0",
|
||||
sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
|
||||
)
|
||||
|
||||
# Please run
|
||||
# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
|
||||
# libopencv-calib3d-dev libopencv-features2d-dev \
|
||||
# libopencv-imgproc-dev libopencv-video-dev
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
# Please run $ brew install opencv
|
||||
new_local_repository(
|
||||
name = "linux_ffmpeg",
|
||||
build_file = "@//third_party:ffmpeg_linux.BUILD",
|
||||
path = "/usr"
|
||||
)
|
||||
|
||||
# Please run $ brew install opencv@3
|
||||
new_local_repository(
|
||||
name = "macos_opencv",
|
||||
build_file = "@//third_party:opencv_macos.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "macos_ffmpeg",
|
||||
build_file = "@//third_party:ffmpeg_macos.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "android_opencv",
|
||||
sha256="056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b",
|
||||
build_file = "@//third_party:opencv_android.BUILD",
|
||||
strip_prefix = "OpenCV-android-sdk",
|
||||
type = "zip",
|
||||
url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip",
|
||||
url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip",
|
||||
)
|
||||
|
||||
# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
|
||||
# trigger duplicate symbol errors in the linking stage of building a mediapipe ios app.
|
||||
# To get a higher version of OpenCV for iOS, opencv2.framework needs to be built from source with
|
||||
# '-DBUILD_PROTOBUF=OFF -DBUILD_opencv_dnn=OFF'.
|
||||
http_archive(
|
||||
name = "ios_opencv",
|
||||
sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2",
|
||||
build_file = "@//third_party:opencv_ios.BUILD",
|
||||
type = "zip",
|
||||
url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
|
||||
)
|
||||
|
||||
RULES_JVM_EXTERNAL_TAG = "2.2"
|
||||
@@ -132,11 +216,19 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
|
||||
|
||||
maven_install(
|
||||
artifacts = [
|
||||
"com.android.support.constraint:constraint-layout:aar:1.0.2",
|
||||
"androidx.appcompat:appcompat:aar:1.0.2",
|
||||
"androidx.annotation:annotation:aar:1.1.0",
|
||||
"androidx.appcompat:appcompat:aar:1.1.0-rc01",
|
||||
"androidx.camera:camera-core:aar:1.0.0-alpha06",
|
||||
"androidx.camera:camera-camera2:aar:1.0.0-alpha06",
|
||||
"androidx.constraintlayout:constraintlayout:aar:1.1.3",
|
||||
"androidx.core:core:aar:1.1.0-rc03",
|
||||
"androidx.legacy:legacy-support-v4:aar:1.0.0",
|
||||
"androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
|
||||
"com.google.android.material:material:aar:1.0.0-rc01",
|
||||
],
|
||||
repositories = [
|
||||
"https://dl.google.com/dl/android/maven2",
|
||||
"https://repo1.maven.org/maven2",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -153,10 +245,10 @@ maven_jar(
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
name = "androidx_concurrent_futures",
|
||||
artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
|
||||
server = "google_server",
|
||||
name = "androidx_concurrent_futures",
|
||||
artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
|
||||
server = "google_server",
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
@@ -191,3 +283,44 @@ android_ndk_repository(
|
||||
android_sdk_repository(
|
||||
name = "androidsdk",
|
||||
)
|
||||
|
||||
# iOS basic build deps.
|
||||
|
||||
http_archive(
|
||||
name = "build_bazel_rules_apple",
|
||||
sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e",
|
||||
strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3",
|
||||
url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
|
||||
)
|
||||
|
||||
load(
|
||||
"@build_bazel_rules_apple//apple:repositories.bzl",
|
||||
"apple_rules_dependencies",
|
||||
)
|
||||
|
||||
apple_rules_dependencies()
|
||||
|
||||
load(
|
||||
"@build_bazel_rules_swift//swift:repositories.bzl",
|
||||
"swift_rules_dependencies",
|
||||
)
|
||||
|
||||
swift_rules_dependencies()
|
||||
|
||||
load(
|
||||
"@build_bazel_apple_support//lib:repositories.bzl",
|
||||
"apple_support_dependencies",
|
||||
)
|
||||
|
||||
apple_support_dependencies()
|
||||
|
||||
# More iOS deps.
|
||||
|
||||
http_archive(
|
||||
name = "google_toolbox_for_mac",
|
||||
url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip",
|
||||
sha256 = "e3ac053813c989a88703556df4dc4466e424e30d32108433ed6beaec76ba4fdc",
|
||||
strip_prefix = "google-toolbox-for-mac-2.2.1",
|
||||
build_file = "@//third_party:google_toolbox_for_mac.BUILD",
|
||||
)
|
||||
|
||||
|
||||
+63
-1
@@ -65,11 +65,73 @@ config_setting(
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
config_setting(
|
||||
# Note: this cannot just match "apple_platform_type": "macos" because that option
|
||||
# defaults to "macos" even when building on Linux!
|
||||
alias(
|
||||
name = "macos",
|
||||
actual = select({
|
||||
":macos_i386": ":macos_i386",
|
||||
":macos_x86_64": ":macos_x86_64",
|
||||
"//conditions:default": ":macos_i386", # Arbitrarily chosen from above.
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# Note: this also matches on crosstool_top so that it does not produce ambiguous
|
||||
# selectors when used together with "android".
|
||||
config_setting(
|
||||
name = "ios",
|
||||
values = {
|
||||
"crosstool_top": "@bazel_tools//tools/cpp:toolchain",
|
||||
"apple_platform_type": "ios",
|
||||
},
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
alias(
|
||||
name = "apple",
|
||||
actual = select({
|
||||
":macos": ":macos",
|
||||
":ios": ":ios",
|
||||
"//conditions:default": ":ios", # Arbitrarily chosen from above.
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "macos_i386",
|
||||
values = {
|
||||
"apple_platform_type": "macos",
|
||||
"cpu": "darwin",
|
||||
},
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "macos_x86_64",
|
||||
values = {
|
||||
"apple_platform_type": "macos",
|
||||
"cpu": "darwin_x86_64",
|
||||
},
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
[
|
||||
config_setting(
|
||||
name = arch,
|
||||
values = {"cpu": arch},
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
for arch in [
|
||||
"ios_i386",
|
||||
"ios_x86_64",
|
||||
"ios_armv7",
|
||||
"ios_arm64",
|
||||
"ios_arm64e",
|
||||
]
|
||||
]
|
||||
|
||||
exports_files(
|
||||
["provisioning_profile.mobileprovision"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
{
|
||||
"additionalFilePaths" : [
|
||||
"/BUILD",
|
||||
"mediapipe/BUILD",
|
||||
"mediapipe/objc/BUILD",
|
||||
"mediapipe/examples/ios/BUILD",
|
||||
"mediapipe/examples/ios/edgedetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/facedetectioncpu/BUILD",
|
||||
"mediapipe/examples/ios/facedetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/handdetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/handtrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/objectdetectioncpu/BUILD",
|
||||
"mediapipe/examples/ios/objectdetectiongpu/BUILD"
|
||||
],
|
||||
"buildTargets" : [
|
||||
"//mediapipe/examples/ios/edgedetectiongpu:EdgeDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp",
|
||||
"//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp",
|
||||
"//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
|
||||
"//mediapipe/objc:mediapipe_framework_ios"
|
||||
],
|
||||
"optionSet" : {
|
||||
"BazelBuildOptionsDebug" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"BazelBuildOptionsRelease" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"BazelBuildStartupOptionsDebug" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"BazelBuildStartupOptionsRelease" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"BuildActionPostActionScript" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"BuildActionPreActionScript" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"CommandlineArguments" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"EnvironmentVariables" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"LaunchActionPostActionScript" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"LaunchActionPreActionScript" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"ProjectGenerationBazelStartupOptions" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"TestActionPostActionScript" : {
|
||||
"p" : "$(inherited)"
|
||||
},
|
||||
"TestActionPreActionScript" : {
|
||||
"p" : "$(inherited)"
|
||||
}
|
||||
},
|
||||
"projectName" : "Mediapipe",
|
||||
"sourceFilters" : [
|
||||
"mediapipe",
|
||||
"mediapipe/calculators",
|
||||
"mediapipe/calculators/core",
|
||||
"mediapipe/calculators/image",
|
||||
"mediapipe/calculators/internal",
|
||||
"mediapipe/calculators/tflite",
|
||||
"mediapipe/calculators/util",
|
||||
"mediapipe/examples",
|
||||
"mediapipe/examples/ios",
|
||||
"mediapipe/examples/ios/edgedetectiongpu",
|
||||
"mediapipe/examples/ios/edgedetectiongpu/Base.lproj",
|
||||
"mediapipe/examples/ios/facedetectioncpu",
|
||||
"mediapipe/examples/ios/facedetectioncpu/Base.lproj",
|
||||
"mediapipe/examples/ios/facedetectiongpu",
|
||||
"mediapipe/examples/ios/facedetectiongpu/Base.lproj",
|
||||
"mediapipe/examples/ios/handdetectiongpu",
|
||||
"mediapipe/examples/ios/handdetectiongpu/Base.lproj",
|
||||
"mediapipe/examples/ios/handtrackinggpu",
|
||||
"mediapipe/examples/ios/handtrackinggpu/Base.lproj",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu/Base.lproj",
|
||||
"mediapipe/examples/ios/objectdetectioncpu",
|
||||
"mediapipe/examples/ios/objectdetectioncpu/Base.lproj",
|
||||
"mediapipe/examples/ios/objectdetectiongpu",
|
||||
"mediapipe/examples/ios/objectdetectiongpu/Base.lproj",
|
||||
"mediapipe/framework",
|
||||
"mediapipe/framework/deps",
|
||||
"mediapipe/framework/formats",
|
||||
"mediapipe/framework/formats/annotation",
|
||||
"mediapipe/framework/formats/object_detection",
|
||||
"mediapipe/framework/port",
|
||||
"mediapipe/framework/profiler",
|
||||
"mediapipe/framework/stream_handler",
|
||||
"mediapipe/framework/tool",
|
||||
"mediapipe/gpu",
|
||||
"mediapipe/graphs",
|
||||
"mediapipe/graphs/edge_detection",
|
||||
"mediapipe/graphs/face_detection",
|
||||
"mediapipe/graphs/hand_tracking",
|
||||
"mediapipe/graphs/object_detection",
|
||||
"mediapipe/models",
|
||||
"mediapipe/objc",
|
||||
"mediapipe/util",
|
||||
"mediapipe/util/android",
|
||||
"mediapipe/util/android/file",
|
||||
"mediapipe/util/android/file/base",
|
||||
"mediapipe/util/tflite",
|
||||
"mediapipe/util/tflite/operations"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"configDefaults" : {
|
||||
"optionSet" : {
|
||||
"CLANG_CXX_LANGUAGE_STANDARD" : {
|
||||
"p" : "c++14"
|
||||
}
|
||||
}
|
||||
},
|
||||
"packages" : [
|
||||
"",
|
||||
"mediapipe",
|
||||
"mediapipe/objc",
|
||||
"mediapipe/examples/ios",
|
||||
"mediapipe/examples/ios/edgedetectiongpu",
|
||||
"mediapipe/examples/ios/facedetectioncpu",
|
||||
"mediapipe/examples/ios/facedetectiongpu",
|
||||
"mediapipe/examples/ios/handdetectiongpu",
|
||||
"mediapipe/examples/ios/handtrackinggpu",
|
||||
"mediapipe/examples/ios/multihandtrackinggpu",
|
||||
"mediapipe/examples/ios/objectdetectioncpu",
|
||||
"mediapipe/examples/ios/objectdetectiongpu"
|
||||
],
|
||||
"projectName" : "Mediapipe",
|
||||
"workspaceRoot" : "../.."
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
|
||||
proto_library(
|
||||
name = "mfcc_mel_calculators_proto",
|
||||
srcs = ["mfcc_mel_calculators.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "mfcc_mel_calculators_cc_proto",
|
||||
srcs = ["mfcc_mel_calculators.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":mfcc_mel_calculators_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "rational_factor_resample_calculator_proto",
|
||||
srcs = ["rational_factor_resample_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "rational_factor_resample_calculator_cc_proto",
|
||||
srcs = ["rational_factor_resample_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":rational_factor_resample_calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "spectrogram_calculator_proto",
|
||||
srcs = ["spectrogram_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "spectrogram_calculator_cc_proto",
|
||||
srcs = ["spectrogram_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":spectrogram_calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "stabilized_log_calculator_proto",
|
||||
srcs = ["stabilized_log_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "stabilized_log_calculator_cc_proto",
|
||||
srcs = ["stabilized_log_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":stabilized_log_calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "time_series_framer_calculator_proto",
|
||||
srcs = ["time_series_framer_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "time_series_framer_calculator_cc_proto",
|
||||
srcs = ["time_series_framer_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":time_series_framer_calculator_proto"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "audio_decoder_calculator",
|
||||
srcs = ["audio_decoder_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:audio_decoder",
|
||||
"//mediapipe/util:audio_decoder_cc_proto",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "basic_time_series_calculators",
|
||||
srcs = ["basic_time_series_calculators.cc"],
|
||||
hdrs = ["basic_time_series_calculators.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "mfcc_mel_calculators",
|
||||
srcs = ["mfcc_mel_calculators.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":mfcc_mel_calculators_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_audio_tools//audio/dsp/mfcc",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "rational_factor_resample_calculator",
|
||||
srcs = ["rational_factor_resample_calculator.cc"],
|
||||
hdrs = ["rational_factor_resample_calculator.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":rational_factor_resample_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_audio_tools//audio/dsp:resampler",
|
||||
"@com_google_audio_tools//audio/dsp:resampler_rational_factor",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "stabilized_log_calculator",
|
||||
srcs = ["stabilized_log_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":stabilized_log_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:core_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_util",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "spectrogram_calculator",
|
||||
srcs = ["spectrogram_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":spectrogram_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:core_proto",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:source_location",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@com_google_audio_tools//audio/dsp/spectrogram",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "time_series_framer_calculator",
|
||||
srcs = ["time_series_framer_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":time_series_framer_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "audio_decoder_calculator_test",
|
||||
srcs = ["audio_decoder_calculator_test.cc"],
|
||||
data = ["//mediapipe/calculators/audio/testdata:test_audios"],
|
||||
deps = [
|
||||
":audio_decoder_calculator",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "basic_time_series_calculators_test",
|
||||
srcs = ["basic_time_series_calculators_test.cc"],
|
||||
deps = [
|
||||
":basic_time_series_calculators",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "mfcc_mel_calculators_test",
|
||||
srcs = ["mfcc_mel_calculators_test.cc"],
|
||||
deps = [
|
||||
":mfcc_mel_calculators",
|
||||
":mfcc_mel_calculators_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "spectrogram_calculator_test",
|
||||
srcs = ["spectrogram_calculator_test.cc"],
|
||||
deps = [
|
||||
":spectrogram_calculator",
|
||||
":spectrogram_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:benchmark",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@com_google_audio_tools//audio/dsp:number_util",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "stabilized_log_calculator_test",
|
||||
srcs = ["stabilized_log_calculator_test.cc"],
|
||||
deps = [
|
||||
":stabilized_log_calculator",
|
||||
":stabilized_log_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "time_series_framer_calculator_test",
|
||||
srcs = ["time_series_framer_calculator_test.cc"],
|
||||
deps = [
|
||||
":time_series_framer_calculator",
|
||||
":time_series_framer_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "rational_factor_resample_calculator_test",
|
||||
srcs = ["rational_factor_resample_calculator_test.cc"],
|
||||
deps = [
|
||||
":rational_factor_resample_calculator",
|
||||
":rational_factor_resample_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@com_google_audio_tools//audio/dsp:signal_vector_util",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,110 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/audio_decoder.h"
|
||||
#include "mediapipe/util/audio_decoder.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// The AudioDecoderCalculator decodes an audio stream of the media file. It
|
||||
// produces two output streams contain audio packets and the header infomation.
|
||||
//
|
||||
// Output Streams:
|
||||
// AUDIO: Output audio frames (Matrix).
|
||||
// AUDIO_HEADER:
|
||||
// Optional audio header information output
|
||||
// Input Side Packets:
|
||||
// INPUT_FILE_PATH: The input file path.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "AudioDecoderCalculator"
|
||||
// input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
// output_stream: "AUDIO:audio"
|
||||
// output_stream: "AUDIO_HEADER:audio_header"
|
||||
// node_options {
|
||||
// [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
// audio_stream { stream_index: 0 }
|
||||
// start_time: 0
|
||||
// end_time: 1
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// TODO: support decoding multiple streams.
|
||||
class AudioDecoderCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
std::unique_ptr<AudioDecoder> decoder_;
|
||||
};
|
||||
|
||||
::mediapipe::Status AudioDecoderCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
cc->InputSidePackets().Tag("INPUT_FILE_PATH").Set<std::string>();
|
||||
if (cc->InputSidePackets().HasTag("OPTIONS")) {
|
||||
cc->InputSidePackets().Tag("OPTIONS").Set<mediapipe::AudioDecoderOptions>();
|
||||
}
|
||||
cc->Outputs().Tag("AUDIO").Set<Matrix>();
|
||||
if (cc->Outputs().HasTag("AUDIO_HEADER")) {
|
||||
cc->Outputs().Tag("AUDIO_HEADER").SetNone();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status AudioDecoderCalculator::Open(CalculatorContext* cc) {
|
||||
const std::string& input_file_path =
|
||||
cc->InputSidePackets().Tag("INPUT_FILE_PATH").Get<std::string>();
|
||||
const auto& decoder_options =
|
||||
tool::RetrieveOptions(cc->Options<mediapipe::AudioDecoderOptions>(),
|
||||
cc->InputSidePackets(), "OPTIONS");
|
||||
decoder_ = absl::make_unique<AudioDecoder>();
|
||||
MP_RETURN_IF_ERROR(decoder_->Initialize(input_file_path, decoder_options));
|
||||
std::unique_ptr<mediapipe::TimeSeriesHeader> header =
|
||||
absl::make_unique<mediapipe::TimeSeriesHeader>();
|
||||
if (decoder_->FillAudioHeader(decoder_options.audio_stream(0), header.get())
|
||||
.ok()) {
|
||||
// Only pass on a header if the decoder could actually produce one.
|
||||
// otherwise, the header will be empty.
|
||||
cc->Outputs().Tag("AUDIO_HEADER").SetHeader(Adopt(header.release()));
|
||||
}
|
||||
cc->Outputs().Tag("AUDIO_HEADER").Close();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status AudioDecoderCalculator::Process(CalculatorContext* cc) {
|
||||
Packet data;
|
||||
int options_index = -1;
|
||||
auto status = decoder_->GetData(&options_index, &data);
|
||||
if (status.ok()) {
|
||||
cc->Outputs().Tag("AUDIO").AddPacket(data);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
::mediapipe::Status AudioDecoderCalculator::Close(CalculatorContext* cc) {
|
||||
return decoder_->Close();
|
||||
}
|
||||
|
||||
REGISTER_CALCULATOR(AudioDecoderCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,149 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, TestWAV) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
output_stream: "AUDIO_HEADER:audio_header"
|
||||
node_options {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
"/mediapipe/calculators/audio/"
|
||||
"testdata/sine_wave_1k_44100_mono_2_sec_wav.audio"));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
MP_EXPECT_OK(runner.Outputs()
|
||||
.Tag("AUDIO_HEADER")
|
||||
.header.ValidateAsType<mediapipe::TimeSeriesHeader>());
|
||||
const mediapipe::TimeSeriesHeader& header =
|
||||
runner.Outputs()
|
||||
.Tag("AUDIO_HEADER")
|
||||
.header.Get<mediapipe::TimeSeriesHeader>();
|
||||
EXPECT_EQ(44100, header.sample_rate());
|
||||
EXPECT_EQ(1, header.num_channels());
|
||||
EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >=
|
||||
std::ceil(44100.0 * 2 / 2048));
|
||||
}
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, Test48KWAV) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
output_stream: "AUDIO_HEADER:audio_header"
|
||||
node_options {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
"/mediapipe/calculators/audio/"
|
||||
"testdata/sine_wave_1k_48000_stereo_2_sec_wav.audio"));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
MP_EXPECT_OK(runner.Outputs()
|
||||
.Tag("AUDIO_HEADER")
|
||||
.header.ValidateAsType<mediapipe::TimeSeriesHeader>());
|
||||
const mediapipe::TimeSeriesHeader& header =
|
||||
runner.Outputs()
|
||||
.Tag("AUDIO_HEADER")
|
||||
.header.Get<mediapipe::TimeSeriesHeader>();
|
||||
EXPECT_EQ(48000, header.sample_rate());
|
||||
EXPECT_EQ(2, header.num_channels());
|
||||
EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >=
|
||||
std::ceil(48000.0 * 2 / 1024));
|
||||
}
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, TestMP3) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
output_stream: "AUDIO_HEADER:audio_header"
|
||||
node_options {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
"/mediapipe/calculators/audio/"
|
||||
"testdata/sine_wave_1k_44100_stereo_2_sec_mp3.audio"));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
MP_EXPECT_OK(runner.Outputs()
|
||||
.Tag("AUDIO_HEADER")
|
||||
.header.ValidateAsType<mediapipe::TimeSeriesHeader>());
|
||||
const mediapipe::TimeSeriesHeader& header =
|
||||
runner.Outputs()
|
||||
.Tag("AUDIO_HEADER")
|
||||
.header.Get<mediapipe::TimeSeriesHeader>();
|
||||
EXPECT_EQ(44100, header.sample_rate());
|
||||
EXPECT_EQ(2, header.num_channels());
|
||||
EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >=
|
||||
std::ceil(44100.0 * 2 / 1152));
|
||||
}
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, TestAAC) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
output_stream: "AUDIO_HEADER:audio_header"
|
||||
node_options {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
"/mediapipe/calculators/audio/"
|
||||
"testdata/sine_wave_1k_44100_stereo_2_sec_aac.audio"));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
MP_EXPECT_OK(runner.Outputs()
|
||||
.Tag("AUDIO_HEADER")
|
||||
.header.ValidateAsType<mediapipe::TimeSeriesHeader>());
|
||||
const mediapipe::TimeSeriesHeader& header =
|
||||
runner.Outputs()
|
||||
.Tag("AUDIO_HEADER")
|
||||
.header.Get<mediapipe::TimeSeriesHeader>();
|
||||
EXPECT_EQ(44100, header.sample_rate());
|
||||
EXPECT_EQ(2, header.num_channels());
|
||||
EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >=
|
||||
std::ceil(44100.0 * 2 / 1024));
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,403 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Basic Calculators that operate on TimeSeries streams.
|
||||
#include "mediapipe/calculators/audio/basic_time_series_calculators.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/util/time_series_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
static bool SafeMultiply(int x, int y, int* result) {
|
||||
static_assert(sizeof(int64) >= 2 * sizeof(int),
|
||||
"Unable to detect overflow after multiplication");
|
||||
const int64 big = static_cast<int64>(x) * static_cast<int64>(y);
|
||||
if (big > static_cast<int64>(INT_MIN) && big < static_cast<int64>(INT_MAX)) {
|
||||
if (result != nullptr) *result = static_cast<int>(big);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
::mediapipe::Status BasicTimeSeriesCalculatorBase::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<Matrix>(
|
||||
// Input stream with TimeSeriesHeader.
|
||||
);
|
||||
cc->Outputs().Index(0).Set<Matrix>(
|
||||
// Output stream with TimeSeriesHeader.
|
||||
);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status BasicTimeSeriesCalculatorBase::Open(CalculatorContext* cc) {
|
||||
TimeSeriesHeader input_header;
|
||||
MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
|
||||
cc->Inputs().Index(0).Header(), &input_header));
|
||||
|
||||
auto output_header = new TimeSeriesHeader(input_header);
|
||||
MP_RETURN_IF_ERROR(MutateHeader(output_header));
|
||||
cc->Outputs().Index(0).SetHeader(Adopt(output_header));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status BasicTimeSeriesCalculatorBase::Process(
|
||||
CalculatorContext* cc) {
|
||||
const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
|
||||
MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
|
||||
input, cc->Inputs().Index(0).Header().Get<TimeSeriesHeader>()));
|
||||
|
||||
std::unique_ptr<Matrix> output(new Matrix(ProcessMatrix(input)));
|
||||
MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
|
||||
*output, cc->Outputs().Index(0).Header().Get<TimeSeriesHeader>()));
|
||||
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status BasicTimeSeriesCalculatorBase::MutateHeader(
|
||||
TimeSeriesHeader* output_header) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
// Calculator to sum an input time series across channels. This is
|
||||
// useful for e.g. computing 'summary SAI' pitchogram features.
|
||||
//
|
||||
// Options proto: None.
|
||||
class SumTimeSeriesAcrossChannelsCalculator
|
||||
: public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
|
||||
output_header->set_num_channels(1);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
return input_matrix.colwise().sum();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(SumTimeSeriesAcrossChannelsCalculator);
|
||||
|
||||
// Calculator to average an input time series across channels. This is
|
||||
// useful for e.g. converting stereo or multi-channel files to mono.
|
||||
//
|
||||
// Options proto: None.
|
||||
class AverageTimeSeriesAcrossChannelsCalculator
|
||||
: public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
|
||||
output_header->set_num_channels(1);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
return input_matrix.colwise().mean();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(AverageTimeSeriesAcrossChannelsCalculator);
|
||||
|
||||
// Calculator to convert a (temporal) summary SAI stream (a single-channel
|
||||
// stream output by SumTimeSeriesAcrossChannelsCalculator) into pitchogram
|
||||
// frames by transposing the input packets, swapping the time and channel axes.
|
||||
//
|
||||
// Options proto: None.
|
||||
class SummarySaiToPitchogramCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
|
||||
if (output_header->num_channels() != 1) {
|
||||
return tool::StatusInvalid(
|
||||
absl::StrCat("Expected single-channel input, got ",
|
||||
output_header->num_channels()));
|
||||
}
|
||||
output_header->set_num_channels(output_header->num_samples());
|
||||
output_header->set_num_samples(1);
|
||||
output_header->set_sample_rate(output_header->packet_rate());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
return input_matrix.transpose();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(SummarySaiToPitchogramCalculator);
|
||||
|
||||
// Calculator to reverse the order of channels in TimeSeries packets.
|
||||
// This is useful for e.g. interfacing with the speech pipeline which uses the
|
||||
// opposite convention to the hearing filterbanks.
|
||||
//
|
||||
// Options proto: None.
|
||||
class ReverseChannelOrderCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
return input_matrix.colwise().reverse();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(ReverseChannelOrderCalculator);
|
||||
|
||||
// Calculator to flatten all samples in a TimeSeries packet down into
|
||||
// a single 'sample' vector. This is useful for e.g. stacking several
|
||||
// frames of features into a single feature vector.
|
||||
//
|
||||
// Options proto: None.
|
||||
class FlattenPacketCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
|
||||
const int num_input_channels = output_header->num_channels();
|
||||
const int num_input_samples = output_header->num_samples();
|
||||
RET_CHECK(num_input_channels >= 0)
|
||||
<< "FlattenPacketCalculator: num_input_channels < 0";
|
||||
RET_CHECK(num_input_samples >= 0)
|
||||
<< "FlattenPacketCalculator: num_input_samples < 0";
|
||||
int output_num_channels;
|
||||
RET_CHECK(SafeMultiply(num_input_channels, num_input_samples,
|
||||
&output_num_channels))
|
||||
<< "FlattenPacketCalculator: Multiplication failed.";
|
||||
output_header->set_num_channels(output_num_channels);
|
||||
output_header->set_num_samples(1);
|
||||
output_header->set_sample_rate(output_header->packet_rate());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
// Flatten by interleaving channels so that full samples are
|
||||
// stacked on top of each other instead of interleaving samples
|
||||
// from the same channel.
|
||||
Matrix output(input_matrix.size(), 1);
|
||||
for (int sample = 0; sample < input_matrix.cols(); ++sample) {
|
||||
output.middleRows(sample * input_matrix.rows(), input_matrix.rows()) =
|
||||
input_matrix.col(sample);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(FlattenPacketCalculator);
|
||||
|
||||
// Calculator to subtract the within-packet mean for each channel from each
|
||||
// corresponding channel.
|
||||
//
|
||||
// Options proto: None.
|
||||
class SubtractMeanCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
Matrix mean = input_matrix.rowwise().mean();
|
||||
return input_matrix - mean.replicate(1, input_matrix.cols());
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(SubtractMeanCalculator);
|
||||
|
||||
// Calculator to subtract the mean over all values (across all times and
|
||||
// channels) in a Packet from the values in that Packet.
|
||||
//
|
||||
// Options proto: None.
|
||||
class SubtractMeanAcrossChannelsCalculator
|
||||
: public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
auto mean = input_matrix.mean();
|
||||
return (input_matrix.array() - mean).matrix();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(SubtractMeanAcrossChannelsCalculator);
|
||||
|
||||
// Calculator to divide all values in a Packet by the average value across all
|
||||
// times and channels in the packet. This is useful for normalizing
|
||||
// nonnegative quantities like power, but might cause unexpected results if used
|
||||
// with Packets that can contain negative numbers.
|
||||
//
|
||||
// If mean is exactly zero, the output will be a matrix of all ones, because
|
||||
// that's what happens in other cases where all values are equal.
|
||||
//
|
||||
// Options proto: None.
|
||||
class DivideByMeanAcrossChannelsCalculator
|
||||
: public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
auto mean = input_matrix.mean();
|
||||
|
||||
if (mean != 0) {
|
||||
return input_matrix / mean;
|
||||
|
||||
// When used with nonnegative matrices, the mean will only be zero if the
|
||||
// entire matrix is exactly zero. If mean is exactly zero, the output will
|
||||
// be a matrix of all ones, because that's what happens in other cases
|
||||
// where
|
||||
// all values are equal.
|
||||
} else {
|
||||
return Matrix::Ones(input_matrix.rows(), input_matrix.cols());
|
||||
}
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(DivideByMeanAcrossChannelsCalculator);
|
||||
|
||||
// Calculator to calculate the mean for each channel.
|
||||
//
|
||||
// Options proto: None.
|
||||
class MeanCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
|
||||
output_header->set_num_samples(1);
|
||||
output_header->set_sample_rate(output_header->packet_rate());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
return input_matrix.rowwise().mean();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(MeanCalculator);
|
||||
|
||||
// Calculator to calculate the uncorrected sample standard deviation in each
|
||||
// channel, independently for each Packet. I.e. divide by the number of samples
|
||||
// in the Packet, not (<number of samples> - 1).
|
||||
//
|
||||
// Options proto: None.
|
||||
class StandardDeviationCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
|
||||
output_header->set_num_samples(1);
|
||||
output_header->set_sample_rate(output_header->packet_rate());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
Eigen::VectorXf mean = input_matrix.rowwise().mean();
|
||||
return (input_matrix.colwise() - mean).rowwise().norm() /
|
||||
sqrt(input_matrix.cols());
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(StandardDeviationCalculator);
|
||||
|
||||
// Calculator to calculate the covariance matrix. If the input matrix
|
||||
// has N channels, the output matrix will be an N by N symmetric
|
||||
// matrix.
|
||||
//
|
||||
// Options proto: None.
|
||||
class CovarianceCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
|
||||
output_header->set_num_samples(output_header->num_channels());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
auto mean = input_matrix.rowwise().mean();
|
||||
auto zero_mean_input =
|
||||
input_matrix - mean.replicate(1, input_matrix.cols());
|
||||
return (zero_mean_input * zero_mean_input.transpose()) /
|
||||
input_matrix.cols();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(CovarianceCalculator);
|
||||
|
||||
// Calculator to get the per column L2 norm of an input time series.
|
||||
//
|
||||
// Options proto: None.
|
||||
class L2NormCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
|
||||
output_header->set_num_channels(1);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
return input_matrix.colwise().norm();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(L2NormCalculator);
|
||||
|
||||
// Calculator to convert each column of a matrix to a unit vector.
|
||||
//
|
||||
// Options proto: None.
|
||||
class L2NormalizeColumnCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
return input_matrix.colwise().normalized();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(L2NormalizeColumnCalculator);
|
||||
|
||||
// Calculator to apply L2 normalization to the input matrix.
|
||||
//
|
||||
// Returns the matrix as is if the RMS is <= 1E-8.
|
||||
// Options proto: None.
|
||||
class L2NormalizeCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
constexpr double kEpsilon = 1e-8;
|
||||
double rms = std::sqrt(input_matrix.array().square().mean());
|
||||
if (rms <= kEpsilon) {
|
||||
return input_matrix;
|
||||
}
|
||||
return input_matrix / rms;
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(L2NormalizeCalculator);
|
||||
|
||||
// Calculator to apply Peak normalization to the input matrix.
|
||||
//
|
||||
// Returns the matrix as is if the peak is <= 1E-8.
|
||||
// Options proto: None.
|
||||
class PeakNormalizeCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
constexpr double kEpsilon = 1e-8;
|
||||
double max_pcm = input_matrix.cwiseAbs().maxCoeff();
|
||||
if (max_pcm <= kEpsilon) {
|
||||
return input_matrix;
|
||||
}
|
||||
return input_matrix / max_pcm;
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(PeakNormalizeCalculator);
|
||||
|
||||
// Calculator to compute the elementwise square of an input time series.
|
||||
//
|
||||
// Options proto: None.
|
||||
class ElementwiseSquareCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
return input_matrix.array().square();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(ElementwiseSquareCalculator);
|
||||
|
||||
// Calculator that outputs first floor(num_samples / 2) of the samples.
|
||||
//
|
||||
// Options proto: None.
|
||||
class FirstHalfSlicerCalculator : public BasicTimeSeriesCalculatorBase {
|
||||
protected:
|
||||
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
|
||||
const int num_input_samples = output_header->num_samples();
|
||||
RET_CHECK(num_input_samples >= 0)
|
||||
<< "FirstHalfSlicerCalculator: num_input_samples < 0";
|
||||
output_header->set_num_samples(num_input_samples / 2);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Matrix ProcessMatrix(const Matrix& input_matrix) final {
|
||||
return input_matrix.block(0, 0, input_matrix.rows(),
|
||||
input_matrix.cols() / 2);
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(FirstHalfSlicerCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Abstract base class for basic MediaPipe calculators that operate on
|
||||
// TimeSeries streams and don't require any Options protos.
|
||||
// Subclasses must override ProcessMatrix, and optionally
|
||||
// MutateHeader.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_AUDIO_BASIC_TIME_SERIES_CALCULATORS_H_
|
||||
#define MEDIAPIPE_CALCULATORS_AUDIO_BASIC_TIME_SERIES_CALCULATORS_H_
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class BasicTimeSeriesCalculatorBase : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
protected:
|
||||
// Open() calls this method to mutate the output stream header. The input
|
||||
// to this function will contain a copy of the input stream header, so
|
||||
// subclasses that do not need to mutate the header do not need to override
|
||||
// it.
|
||||
virtual ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header);
|
||||
|
||||
// Process() calls this method on each packet to compute the output matrix.
|
||||
virtual Matrix ProcessMatrix(const Matrix& input_matrix) = 0;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_AUDIO_BASIC_TIME_SERIES_CALCULATORS_H_
|
||||
@@ -0,0 +1,515 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/util/time_series_test_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class SumTimeSeriesAcrossChannelsCalculatorTest
|
||||
: public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
calculator_name_ = "SumTimeSeriesAcrossChannelsCalculator";
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(SumTimeSeriesAcrossChannelsCalculatorTest, IsNoOpOnSingleChannelInputs) {
|
||||
const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 num_channels: 1 num_samples: 5");
|
||||
const Matrix input =
|
||||
Matrix::Random(header.num_channels(), header.num_samples());
|
||||
|
||||
Test(header, {input}, header, {input});
|
||||
}
|
||||
|
||||
TEST_F(SumTimeSeriesAcrossChannelsCalculatorTest, ConstantPacket) {
|
||||
const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 num_channels: 3 num_samples: 5");
|
||||
TimeSeriesHeader output_header(header);
|
||||
output_header.set_num_channels(1);
|
||||
|
||||
Test(header,
|
||||
{Matrix::Constant(header.num_channels(), header.num_samples(), 1)},
|
||||
output_header,
|
||||
{Matrix::Constant(1, header.num_samples(), header.num_channels())});
|
||||
}
|
||||
|
||||
TEST_F(SumTimeSeriesAcrossChannelsCalculatorTest, MultiplePackets) {
|
||||
const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 num_channels: 3 num_samples: 5");
|
||||
Matrix in(header.num_channels(), header.num_samples());
|
||||
in << 10, -1, -1, 0, 0, 20, -2, 0, 1, 0, 30, -3, 1, 0, 12;
|
||||
|
||||
TimeSeriesHeader output_header(header);
|
||||
output_header.set_num_channels(1);
|
||||
Matrix out(1, header.num_samples());
|
||||
out << 60, -6, 0, 1, 12;
|
||||
|
||||
Test(header, {in, 2 * in, in + Matrix::Constant(in.rows(), in.cols(), 3.5f)},
|
||||
output_header,
|
||||
{out, 2 * out,
|
||||
out + Matrix::Constant(out.rows(), out.cols(),
|
||||
3.5 * header.num_channels())});
|
||||
}
|
||||
|
||||
class AverageTimeSeriesAcrossChannelsCalculatorTest
|
||||
: public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
calculator_name_ = "AverageTimeSeriesAcrossChannelsCalculator";
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(AverageTimeSeriesAcrossChannelsCalculatorTest,
|
||||
IsNoOpOnSingleChannelInputs) {
|
||||
const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 num_channels: 1 num_samples: 5");
|
||||
const Matrix input =
|
||||
Matrix::Random(header.num_channels(), header.num_samples());
|
||||
|
||||
Test(header, {input}, header, {input});
|
||||
}
|
||||
|
||||
TEST_F(AverageTimeSeriesAcrossChannelsCalculatorTest, ConstantPacket) {
|
||||
const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 num_channels: 3 num_samples: 5");
|
||||
TimeSeriesHeader output_header(header);
|
||||
output_header.set_num_channels(1);
|
||||
|
||||
Matrix input =
|
||||
Matrix::Constant(header.num_channels(), header.num_samples(), 0.0);
|
||||
input.row(0) = Matrix::Constant(1, header.num_samples(), 1.0);
|
||||
|
||||
Test(
|
||||
header, {input}, output_header,
|
||||
{Matrix::Constant(1, header.num_samples(), 1.0 / header.num_channels())});
|
||||
}
|
||||
|
||||
class SummarySaiToPitchogramCalculatorTest
|
||||
: public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
calculator_name_ = "SummarySaiToPitchogramCalculator";
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(SummarySaiToPitchogramCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 1 num_samples: 3");
|
||||
Matrix input(1, input_header.num_samples());
|
||||
input << 3, -9, 4;
|
||||
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 5.0 packet_rate: 5.0 num_channels: 3 num_samples: 1");
|
||||
Matrix output(input_header.num_samples(), 1);
|
||||
output << 3, -9, 4;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class ReverseChannelOrderCalculatorTest
|
||||
: public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "ReverseChannelOrderCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(ReverseChannelOrderCalculatorTest, IsNoOpOnSingleChannelInputs) {
|
||||
const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 num_channels: 1 num_samples: 5");
|
||||
const Matrix input =
|
||||
Matrix::Random(header.num_channels(), header.num_samples());
|
||||
|
||||
Test(header, {input}, header, {input});
|
||||
}
|
||||
|
||||
TEST_F(ReverseChannelOrderCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 num_channels: 5 num_samples: 2");
|
||||
Matrix input(header.num_channels(), header.num_samples());
|
||||
input.transpose() << 1, 2, 3, 4, 5, -1, -2, -3, -4, -5;
|
||||
Matrix output(header.num_channels(), header.num_samples());
|
||||
output.transpose() << 5, 4, 3, 2, 1, -5, -4, -3, -2, -1;
|
||||
|
||||
Test(header, {input}, header, {output});
|
||||
}
|
||||
|
||||
class FlattenPacketCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "FlattenPacketCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(FlattenPacketCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 2");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input.transpose() << 1, 2, 3, 4, 5, -1, -2, -3, -4, -5;
|
||||
Matrix output(10, 1);
|
||||
output << 1, 2, 3, 4, 5, -1, -2, -3, -4, -5;
|
||||
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 10.0 packet_rate: 10.0 num_channels: 10 num_samples: 1");
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class SubtractMeanCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "SubtractMeanCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(SubtractMeanCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 2");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
Matrix output(input_header.num_channels(), input_header.num_samples());
|
||||
|
||||
// clang-format off
|
||||
input.transpose() << 1, 0, 3, 0, 1,
|
||||
-1, -2, -3, 4, 7;
|
||||
output.transpose() << 1, 1, 3, -2, -3,
|
||||
-1, -1, -3, 2, 3;
|
||||
// clang-format on
|
||||
|
||||
const TimeSeriesHeader output_header = input_header;
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class SubtractMeanAcrossChannelsCalculatorTest
|
||||
: public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
calculator_name_ = "SubtractMeanAcrossChannelsCalculator";
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(SubtractMeanAcrossChannelsCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
|
||||
TimeSeriesHeader output_header(input_header);
|
||||
output_header.set_num_samples(2);
|
||||
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
|
||||
// clang-format off
|
||||
input.transpose() << 1.0, 2.0, 3.0,
|
||||
4.0, 5.0, 6.0;
|
||||
output.transpose() << 1.0 - 3.5, 2.0 - 3.5, 3.0 - 3.5,
|
||||
4.0 - 3.5, 5.0 - 3.5, 6.0 - 3.5;
|
||||
// clang-format on
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class DivideByMeanAcrossChannelsCalculatorTest
|
||||
: public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
calculator_name_ = "DivideByMeanAcrossChannelsCalculator";
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(DivideByMeanAcrossChannelsCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input.transpose() << 1.0, 2.0, 3.0, 4.0, 5.0, 6.0;
|
||||
|
||||
TimeSeriesHeader output_header(input_header);
|
||||
output_header.set_num_samples(2);
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output.transpose() << 1.0 / 3.5, 2.0 / 3.5, 3.0 / 3.5, 4.0 / 3.5, 5.0 / 3.5,
|
||||
6.0 / 3.5;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
TEST_F(DivideByMeanAcrossChannelsCalculatorTest, ReturnsOneForZeroMean) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input.transpose() << -3.0, -2.0, -1.0, 1.0, 2.0, 3.0;
|
||||
|
||||
TimeSeriesHeader output_header(input_header);
|
||||
output_header.set_num_samples(2);
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output.transpose() << 1.0, 1.0, 1.0, 1.0, 1.0, 1.0;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class MeanCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "MeanCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(MeanCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input.transpose() << 1.0, 2.0, 3.0, 4.0, 5.0, 6.0;
|
||||
|
||||
TimeSeriesHeader output_header(input_header);
|
||||
output_header.set_num_samples(1);
|
||||
output_header.set_sample_rate(10.0);
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output << (1.0 + 4.0) / 2, (2.0 + 5.0) / 2, (3.0 + 6.0) / 2;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class StandardDeviationCalculatorTest
|
||||
: public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "StandardDeviationCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(StandardDeviationCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input.transpose() << 0.0, 2.0, 3.0, 4.0, 5.0, 8.0;
|
||||
|
||||
TimeSeriesHeader output_header(input_header);
|
||||
output_header.set_sample_rate(10.0);
|
||||
output_header.set_num_samples(1);
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output << sqrt((pow(0.0 - 2.0, 2) + pow(4.0 - 2.0, 2)) / 2),
|
||||
sqrt((pow(2.0 - 3.5, 2) + pow(5.0 - 3.5, 2)) / 2),
|
||||
sqrt((pow(3.0 - 5.5, 2) + pow(8.0 - 5.5, 2)) / 2);
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class CovarianceCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "CovarianceCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(CovarianceCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
|
||||
// We'll specify in transposed form so we can write one channel at a time.
|
||||
input << 1.0, 3.0, 5.0, 9.0, -1.0, -3.0;
|
||||
|
||||
TimeSeriesHeader output_header(input_header);
|
||||
output_header.set_num_samples(output_header.num_channels());
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output << 1, 2, -1, 2, 4, -2, -1, -2, 1;
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class L2NormCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "L2NormCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(L2NormCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input << 3, 5, 8, 4, 12, -15;
|
||||
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 1 num_samples: 3");
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output << 5, 13, 17;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class L2NormalizeColumnCalculatorTest
|
||||
: public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "L2NormalizeColumnCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(L2NormalizeColumnCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input << 0.3, 0.4, 0.8, 0.5, 0.9, 0.8;
|
||||
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3");
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
|
||||
// The values in output are column-wise L2 normalized
|
||||
// e.g.
|
||||
// |a| -> |a/sqrt(a^2 + b^2)|
|
||||
// |b| |b/sqrt(a^2 + b^2)|
|
||||
output << 0.51449579000473022, 0.40613847970962524, 0.70710676908493042,
|
||||
0.85749292373657227, 0.91381156444549561, 0.70710676908493042;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class L2NormalizeCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "L2NormalizeCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(L2NormalizeCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input << 0.3, 0.4, 0.8, 0.5, 0.9, 0.8;
|
||||
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3");
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
|
||||
// The values in output are L2 normalized
|
||||
// a -> a/sqrt(a^2 + b^2 + c^2 + ...) * sqrt(matrix.cols()*matrix.rows())
|
||||
output << 0.45661166, 0.60881555, 1.21763109, 0.76101943, 1.36983498,
|
||||
1.21763109;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
TEST_F(L2NormalizeCalculatorTest, UnitMatrixStaysUnchanged) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 3 num_samples: 5");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input << 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
|
||||
1.0, -1.0, 1.0;
|
||||
|
||||
Test(input_header, {input}, input_header, {input});
|
||||
}
|
||||
|
||||
class PeakNormalizeCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "PeakNormalizeCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(PeakNormalizeCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input << 0.3, 0.4, 0.8, 0.5, 0.9, 0.8;
|
||||
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3");
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output << 0.33333333, 0.44444444, 0.88888889, 0.55555556, 1.0, 0.88888889;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
TEST_F(PeakNormalizeCalculatorTest, UnitMatrixStaysUnchanged) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 3 num_samples: 5");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input << 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
|
||||
1.0, -1.0, 1.0;
|
||||
|
||||
Test(input_header, {input}, input_header, {input});
|
||||
}
|
||||
|
||||
class ElementwiseSquareCalculatorTest
|
||||
: public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "ElementwiseSquareCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(ElementwiseSquareCalculatorTest, SinglePacket) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
input << 3, 5, 8, 4, 12, -15;
|
||||
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3");
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output << 9, 25, 64, 16, 144, 225;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
class FirstHalfSlicerCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "FirstHalfSlicerCalculator"; }
|
||||
};
|
||||
|
||||
TEST_F(FirstHalfSlicerCalculatorTest, SinglePacketEvenNumSamples) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 2");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
// clang-format off
|
||||
input.transpose() << 0, 1, 2, 3, 4,
|
||||
5, 6, 7, 8, 9;
|
||||
// clang-format on
|
||||
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 1");
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output.transpose() << 0, 1, 2, 3, 4;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
TEST_F(FirstHalfSlicerCalculatorTest, SinglePacketOddNumSamples) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 3");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
// clang-format off
|
||||
input.transpose() << 0, 1, 2, 3, 4,
|
||||
5, 6, 7, 8, 9,
|
||||
0, 0, 0, 0, 0;
|
||||
// clang-format on
|
||||
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 1");
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output.transpose() << 0, 1, 2, 3, 4;
|
||||
|
||||
Test(input_header, {input}, output_header, {output});
|
||||
}
|
||||
|
||||
TEST_F(FirstHalfSlicerCalculatorTest, MultiplePackets) {
|
||||
const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 2");
|
||||
Matrix input(input_header.num_channels(), input_header.num_samples());
|
||||
// clang-format off
|
||||
input.transpose() << 0, 1, 2, 3, 4,
|
||||
5, 6, 7, 8, 9;
|
||||
// clang-format on
|
||||
const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
|
||||
"sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 1");
|
||||
Matrix output(output_header.num_channels(), output_header.num_samples());
|
||||
output.transpose() << 0, 1, 2, 3, 4;
|
||||
|
||||
Test(input_header,
|
||||
{input, 2 * input,
|
||||
input + Matrix::Constant(input.rows(), input.cols(), 3.5f)},
|
||||
output_header,
|
||||
{output, 2 * output,
|
||||
output + Matrix::Constant(output.rows(), output.cols(), 3.5f)});
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,275 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// MediaPipe Calculator wrapper around audio/dsp/mfcc/
|
||||
// classes MelFilterbank (magnitude spectrograms warped to the Mel
|
||||
// approximation of the auditory frequency scale) and Mfcc (Mel Frequency
|
||||
// Cepstral Coefficients, the decorrelated transform of log-Mel-spectrum
|
||||
// commonly used as acoustic features in speech and other audio tasks.
|
||||
// Both calculators expect as input the SQUARED_MAGNITUDE-domain outputs
|
||||
// from the MediaPipe SpectrogramCalculator object.
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "audio/dsp/mfcc/mel_filterbank.h"
|
||||
#include "audio/dsp/mfcc/mfcc.h"
|
||||
#include "mediapipe/calculators/audio/mfcc_mel_calculators.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/time_series_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
// Portable version of TimeSeriesHeader's DebugString.
|
||||
std::string PortableDebugString(const TimeSeriesHeader& header) {
|
||||
std::string unsubstituted_header_debug_str = R"(
|
||||
sample_rate: $0
|
||||
num_channels: $1
|
||||
num_samples: $2
|
||||
packet_rate: $3
|
||||
audio_sample_rate: $4
|
||||
)";
|
||||
return absl::Substitute(unsubstituted_header_debug_str, header.sample_rate(),
|
||||
header.num_channels(), header.num_samples(),
|
||||
header.packet_rate(), header.audio_sample_rate());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// Abstract base class for Calculators that transform feature vectors on a
|
||||
// frame-by-frame basis.
|
||||
// Subclasses must override pure virtual methods ConfigureTransform and
|
||||
// TransformFrame.
|
||||
// Input and output MediaPipe packets are matrices with one column per frame,
|
||||
// and one row per feature dimension. Each input packet results in an
|
||||
// output packet with the same number of columns (but differing numbers of
|
||||
// rows corresponding to the new feature space).
|
||||
class FramewiseTransformCalculatorBase : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<Matrix>(
|
||||
// Sequence of Matrices, each column describing a particular time frame,
|
||||
// each row a feature dimension, with TimeSeriesHeader.
|
||||
);
|
||||
cc->Outputs().Index(0).Set<Matrix>(
|
||||
// Sequence of Matrices, each column describing a particular time frame,
|
||||
// each row a feature dimension, with TimeSeriesHeader.
|
||||
);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
int num_output_channels(void) { return num_output_channels_; }
|
||||
|
||||
void set_num_output_channels(int num_output_channels) {
|
||||
num_output_channels_ = num_output_channels;
|
||||
}
|
||||
|
||||
private:
|
||||
// Takes header and options, and sets up state including calling
|
||||
// set_num_output_channels() on the base object.
|
||||
virtual ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
|
||||
CalculatorContext* cc) = 0;
|
||||
|
||||
// Takes a vector<double> corresponding to an input frame, and
|
||||
// perform the specific transformation to produce an output frame.
|
||||
virtual void TransformFrame(const std::vector<double>& input,
|
||||
std::vector<double>* output) const = 0;
|
||||
|
||||
private:
|
||||
int num_output_channels_;
|
||||
};
|
||||
|
||||
::mediapipe::Status FramewiseTransformCalculatorBase::Open(
|
||||
CalculatorContext* cc) {
|
||||
TimeSeriesHeader input_header;
|
||||
MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
|
||||
cc->Inputs().Index(0).Header(), &input_header));
|
||||
|
||||
::mediapipe::Status status = ConfigureTransform(input_header, cc);
|
||||
|
||||
auto output_header = new TimeSeriesHeader(input_header);
|
||||
output_header->set_num_channels(num_output_channels_);
|
||||
cc->Outputs().Index(0).SetHeader(Adopt(output_header));
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
::mediapipe::Status FramewiseTransformCalculatorBase::Process(
|
||||
CalculatorContext* cc) {
|
||||
const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
|
||||
const int num_frames = input.cols();
|
||||
std::unique_ptr<Matrix> output(new Matrix(num_output_channels_, num_frames));
|
||||
// The main work here is converting each column of the float Matrix
|
||||
// into a vector of doubles, which is what our target functions from
|
||||
// dsp_core consume, and doing the reverse with their output.
|
||||
std::vector<double> input_frame(input.rows());
|
||||
std::vector<double> output_frame(num_output_channels_);
|
||||
|
||||
for (int frame = 0; frame < num_frames; ++frame) {
|
||||
// Copy input from Eigen::Matrix column to vector<float>.
|
||||
Eigen::Map<Eigen::MatrixXd> input_frame_map(&input_frame[0],
|
||||
input_frame.size(), 1);
|
||||
input_frame_map = input.col(frame).cast<double>();
|
||||
|
||||
// Perform the actual transformation.
|
||||
TransformFrame(input_frame, &output_frame);
|
||||
|
||||
// Copy output from vector<float> to Eigen::Vector.
|
||||
CHECK_EQ(output_frame.size(), num_output_channels_);
|
||||
Eigen::Map<const Eigen::MatrixXd> output_frame_map(&output_frame[0],
|
||||
output_frame.size(), 1);
|
||||
output->col(frame) = output_frame_map.cast<float>();
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
// Calculator wrapper around the dsp/mfcc/mfcc.cc routine.
|
||||
// Take frames of squared-magnitude spectra from the SpectrogramCalculator
|
||||
// and convert them into Mel Frequency Cepstral Coefficients.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "MfccCalculator"
|
||||
// input_stream: "spectrogram_frames_stream"
|
||||
// output_stream: "mfcc_frames_stream"
|
||||
// options {
|
||||
// [mediapipe.MfccCalculatorOptions.ext] {
|
||||
// mel_spectrum_params {
|
||||
// channel_count: 20
|
||||
// min_frequency_hertz: 125.0
|
||||
// max_frequency_hertz: 3800.0
|
||||
// }
|
||||
// mfcc_count: 13
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class MfccCalculator : public FramewiseTransformCalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
return FramewiseTransformCalculatorBase::GetContract(cc);
|
||||
}
|
||||
|
||||
private:
|
||||
::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
|
||||
CalculatorContext* cc) override {
|
||||
MfccCalculatorOptions mfcc_options = cc->Options<MfccCalculatorOptions>();
|
||||
mfcc_.reset(new audio_dsp::Mfcc());
|
||||
int input_length = header.num_channels();
|
||||
// Set up the parameters to the Mfcc object.
|
||||
set_num_output_channels(mfcc_options.mfcc_count());
|
||||
mfcc_->set_dct_coefficient_count(num_output_channels());
|
||||
mfcc_->set_upper_frequency_limit(
|
||||
mfcc_options.mel_spectrum_params().max_frequency_hertz());
|
||||
mfcc_->set_lower_frequency_limit(
|
||||
mfcc_options.mel_spectrum_params().min_frequency_hertz());
|
||||
mfcc_->set_filterbank_channel_count(
|
||||
mfcc_options.mel_spectrum_params().channel_count());
|
||||
// An upstream calculator (such as SpectrogramCalculator) must store
|
||||
// the sample rate of its input audio waveform in the TimeSeries Header.
|
||||
// audio_dsp::MelFilterBank needs to know this to
|
||||
// correctly interpret the spectrogram bins.
|
||||
if (!header.has_audio_sample_rate()) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ",
|
||||
PortableDebugString(header)));
|
||||
}
|
||||
// Now we can initialize the Mfcc object.
|
||||
bool initialized =
|
||||
mfcc_->Initialize(input_length, header.audio_sample_rate());
|
||||
|
||||
if (initialized) {
|
||||
return ::mediapipe::OkStatus();
|
||||
} else {
|
||||
return ::mediapipe::Status(mediapipe::StatusCode::kInternal,
|
||||
"Mfcc::Initialize returned uninitialized");
|
||||
}
|
||||
}
|
||||
|
||||
void TransformFrame(const std::vector<double>& input,
|
||||
std::vector<double>* output) const override {
|
||||
mfcc_->Compute(input, output);
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<audio_dsp::Mfcc> mfcc_;
|
||||
};
|
||||
REGISTER_CALCULATOR(MfccCalculator);
|
||||
|
||||
// Calculator wrapper around the dsp/mfcc/mel_filterbank.cc routine.
|
||||
// Take frames of squared-magnitude spectra from the SpectrogramCalculator
|
||||
// and convert them into Mel-warped (linear-magnitude) spectra.
|
||||
// Note: This code computes a mel-frequency filterbank, using a simple
|
||||
// algorithm that gives bad results (some mel channels that are always zero)
|
||||
// if you ask for too many channels.
|
||||
class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
return FramewiseTransformCalculatorBase::GetContract(cc);
|
||||
}
|
||||
|
||||
private:
|
||||
::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
|
||||
CalculatorContext* cc) override {
|
||||
MelSpectrumCalculatorOptions mel_spectrum_options =
|
||||
cc->Options<MelSpectrumCalculatorOptions>();
|
||||
mel_filterbank_.reset(new audio_dsp::MelFilterbank());
|
||||
int input_length = header.num_channels();
|
||||
set_num_output_channels(mel_spectrum_options.channel_count());
|
||||
// An upstream calculator (such as SpectrogramCalculator) must store
|
||||
// the sample rate of its input audio waveform in the TimeSeries Header.
|
||||
// audio_dsp::MelFilterBank needs to know this to
|
||||
// correctly interpret the spectrogram bins.
|
||||
if (!header.has_audio_sample_rate()) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ",
|
||||
PortableDebugString(header)));
|
||||
}
|
||||
bool initialized = mel_filterbank_->Initialize(
|
||||
input_length, header.audio_sample_rate(), num_output_channels(),
|
||||
mel_spectrum_options.min_frequency_hertz(),
|
||||
mel_spectrum_options.max_frequency_hertz());
|
||||
|
||||
if (initialized) {
|
||||
return ::mediapipe::OkStatus();
|
||||
} else {
|
||||
return ::mediapipe::Status(mediapipe::StatusCode::kInternal,
|
||||
"mfcc::Initialize returned uninitialized");
|
||||
}
|
||||
}
|
||||
|
||||
void TransformFrame(const std::vector<double>& input,
|
||||
std::vector<double>* output) const override {
|
||||
mel_filterbank_->Compute(input, output);
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<audio_dsp::MelFilterbank> mel_filterbank_;
|
||||
};
|
||||
REGISTER_CALCULATOR(MelSpectrumCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message MelSpectrumCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional MelSpectrumCalculatorOptions ext = 78581812;
|
||||
}
|
||||
// The fields are to populate the config parameters in
|
||||
// audio/dsp/mfcc/mel_filterbank.h
|
||||
// but the names are chose to mirror
|
||||
// audio/hearing/filterbanks/cochlea_gammatone_filterbank.proto
|
||||
// and the default values match those in
|
||||
// speech/greco3/frontend/filter_bank.proto .
|
||||
|
||||
// Total number of frequency bands to use.
|
||||
optional int32 channel_count = 1 [default = 20];
|
||||
// Lower edge of lowest triangular Mel band.
|
||||
optional float min_frequency_hertz = 2 [default = 125.0];
|
||||
// Upper edge of highest triangular Mel band.
|
||||
optional float max_frequency_hertz = 3 [default = 3800.0];
|
||||
}
|
||||
|
||||
message MfccCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional MfccCalculatorOptions ext = 78450441;
|
||||
}
|
||||
|
||||
// Specification of the underlying mel filterbank.
|
||||
optional MelSpectrumCalculatorOptions mel_spectrum_params = 1;
|
||||
|
||||
// How many MFCC coefficients to emit.
|
||||
optional uint32 mfcc_count = 2 [default = 13];
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "mediapipe/calculators/audio/mfcc_mel_calculators.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/util/time_series_test_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Use a sample rate that is unlikely to be a default somewhere.
|
||||
const float kAudioSampleRate = 8800.0;
|
||||
|
||||
template <typename OptionsType, const char* CalculatorName>
|
||||
class FramewiseTransformCalculatorTest
|
||||
: public TimeSeriesCalculatorTest<OptionsType> {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
this->calculator_name_ = CalculatorName;
|
||||
this->num_input_channels_ = 129;
|
||||
// This is the frame rate coming out of the SpectrogramCalculator.
|
||||
this->input_sample_rate_ = 100.0;
|
||||
}
|
||||
|
||||
// Returns the number of samples per packet.
|
||||
int GenerateRandomNonnegInputStream(int num_packets) {
|
||||
const double kSecondsPerPacket = 0.2;
|
||||
const int num_samples_per_packet =
|
||||
kSecondsPerPacket * this->input_sample_rate_;
|
||||
for (int i = 0; i < num_packets; ++i) {
|
||||
const int timestamp =
|
||||
i * kSecondsPerPacket * Timestamp::kTimestampUnitsPerSecond;
|
||||
// Mfcc, MelSpectrum expect squared-magnitude inputs, so make
|
||||
// sure the input data has no negative values.
|
||||
Matrix* sqdata = this->NewRandomMatrix(this->num_input_channels_,
|
||||
num_samples_per_packet);
|
||||
*sqdata = sqdata->array().square();
|
||||
this->AppendInputPacket(sqdata, timestamp);
|
||||
}
|
||||
return num_samples_per_packet;
|
||||
}
|
||||
|
||||
void CheckOutputPacketMetadata(int expected_num_channels,
|
||||
int expected_num_samples_per_packet) {
|
||||
int expected_timestamp = 0;
|
||||
for (const auto& packet : this->output().packets) {
|
||||
EXPECT_EQ(expected_timestamp, packet.Timestamp().Value());
|
||||
expected_timestamp += expected_num_samples_per_packet /
|
||||
this->input_sample_rate_ *
|
||||
Timestamp::kTimestampUnitsPerSecond;
|
||||
|
||||
const Matrix& output_matrix = packet.template Get<Matrix>();
|
||||
|
||||
EXPECT_EQ(output_matrix.rows(), expected_num_channels);
|
||||
EXPECT_EQ(output_matrix.cols(), expected_num_samples_per_packet);
|
||||
}
|
||||
}
|
||||
|
||||
void SetupGraphAndHeader() {
|
||||
this->InitializeGraph();
|
||||
this->FillInputHeader();
|
||||
}
|
||||
|
||||
// Argument is the expected number of dimensions (channels, columns) in
|
||||
// the output data from the Calculator under test, which the test should
|
||||
// know.
|
||||
void SetupRandomInputPackets() {
|
||||
constexpr int kNumPackets = 5;
|
||||
num_samples_per_packet_ = GenerateRandomNonnegInputStream(kNumPackets);
|
||||
}
|
||||
|
||||
::mediapipe::Status Run() { return this->RunGraph(); }
|
||||
|
||||
void CheckResults(int expected_num_channels) {
|
||||
const auto& output_header =
|
||||
this->output().header.template Get<TimeSeriesHeader>();
|
||||
EXPECT_EQ(this->input_sample_rate_, output_header.sample_rate());
|
||||
CheckOutputPacketMetadata(expected_num_channels, num_samples_per_packet_);
|
||||
|
||||
// Sanity check that output packets have non-zero energy.
|
||||
for (const auto& packet : this->output().packets) {
|
||||
const Matrix& data = packet.template Get<Matrix>();
|
||||
EXPECT_GT(data.squaredNorm(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Allows SetupRandomInputPackets() to inform CheckResults() about how
|
||||
// big the packets are supposed to be.
|
||||
int num_samples_per_packet_;
|
||||
};
|
||||
|
||||
constexpr char kMfccCalculator[] = "MfccCalculator";
|
||||
typedef FramewiseTransformCalculatorTest<MfccCalculatorOptions, kMfccCalculator>
|
||||
MfccCalculatorTest;
|
||||
TEST_F(MfccCalculatorTest, AudioSampleRateFromInputHeader) {
|
||||
audio_sample_rate_ = kAudioSampleRate;
|
||||
SetupGraphAndHeader();
|
||||
SetupRandomInputPackets();
|
||||
|
||||
MP_EXPECT_OK(Run());
|
||||
|
||||
CheckResults(options_.mfcc_count());
|
||||
}
|
||||
TEST_F(MfccCalculatorTest, NoAudioSampleRate) {
|
||||
// Leave audio_sample_rate_ == kUnset, so it is not present in the
|
||||
// input TimeSeriesHeader; expect failure.
|
||||
SetupGraphAndHeader();
|
||||
SetupRandomInputPackets();
|
||||
|
||||
EXPECT_FALSE(Run().ok());
|
||||
}
|
||||
|
||||
constexpr char kMelSpectrumCalculator[] = "MelSpectrumCalculator";
|
||||
typedef FramewiseTransformCalculatorTest<MelSpectrumCalculatorOptions,
|
||||
kMelSpectrumCalculator>
|
||||
MelSpectrumCalculatorTest;
|
||||
TEST_F(MelSpectrumCalculatorTest, AudioSampleRateFromInputHeader) {
|
||||
audio_sample_rate_ = kAudioSampleRate;
|
||||
SetupGraphAndHeader();
|
||||
SetupRandomInputPackets();
|
||||
|
||||
MP_EXPECT_OK(Run());
|
||||
|
||||
CheckResults(options_.channel_count());
|
||||
}
|
||||
TEST_F(MelSpectrumCalculatorTest, NoAudioSampleRate) {
|
||||
// Leave audio_sample_rate_ == kUnset, so it is not present in the
|
||||
// input TimeSeriesHeader; expect failure.
|
||||
SetupGraphAndHeader();
|
||||
SetupRandomInputPackets();
|
||||
|
||||
EXPECT_FALSE(Run().ok());
|
||||
}
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,197 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Defines RationalFactorResampleCalculator.
|
||||
|
||||
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
|
||||
|
||||
#include "audio/dsp/resampler_rational_factor.h"
|
||||
|
||||
using audio_dsp::DefaultResamplingKernel;
|
||||
using audio_dsp::RationalFactorResampler;
|
||||
using audio_dsp::Resampler;
|
||||
|
||||
namespace mediapipe {
|
||||
::mediapipe::Status RationalFactorResampleCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
return ProcessInternal(cc->Inputs().Index(0).Get<Matrix>(), false, cc);
|
||||
}
|
||||
|
||||
::mediapipe::Status RationalFactorResampleCalculator::Close(
|
||||
CalculatorContext* cc) {
|
||||
if (initial_timestamp_ == Timestamp::Unstarted()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
Matrix empty_input_frame(num_channels_, 0);
|
||||
return ProcessInternal(empty_input_frame, true, cc);
|
||||
}
|
||||
|
||||
namespace {
|
||||
void CopyChannelToVector(const Matrix& matrix, int channel,
|
||||
std::vector<float>* vec) {
|
||||
vec->clear();
|
||||
vec->reserve(matrix.cols());
|
||||
for (int sample = 0; sample < matrix.cols(); ++sample) {
|
||||
vec->push_back(matrix(channel, sample));
|
||||
}
|
||||
}
|
||||
|
||||
void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
|
||||
int channel) {
|
||||
if (matrix->cols() == 0) {
|
||||
matrix->resize(matrix->rows(), vec.size());
|
||||
} else {
|
||||
CHECK_EQ(vec.size(), matrix->cols());
|
||||
CHECK_LT(channel, matrix->rows());
|
||||
}
|
||||
for (int sample = 0; sample < matrix->cols(); ++sample) {
|
||||
(*matrix)(channel, sample) = vec[sample];
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
::mediapipe::Status RationalFactorResampleCalculator::Open(
|
||||
CalculatorContext* cc) {
|
||||
RationalFactorResampleCalculatorOptions resample_options =
|
||||
cc->Options<RationalFactorResampleCalculatorOptions>();
|
||||
|
||||
if (!resample_options.has_target_sample_rate()) {
|
||||
return tool::StatusInvalid(
|
||||
"resample_options doesn't have target_sample_rate.");
|
||||
}
|
||||
target_sample_rate_ = resample_options.target_sample_rate();
|
||||
|
||||
TimeSeriesHeader input_header;
|
||||
MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
|
||||
cc->Inputs().Index(0).Header(), &input_header));
|
||||
|
||||
source_sample_rate_ = input_header.sample_rate();
|
||||
num_channels_ = input_header.num_channels();
|
||||
|
||||
// Don't create resamplers for pass-thru (sample rates are equal).
|
||||
if (source_sample_rate_ != target_sample_rate_) {
|
||||
resampler_.resize(num_channels_);
|
||||
for (auto& r : resampler_) {
|
||||
r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_,
|
||||
resample_options);
|
||||
if (!r) {
|
||||
LOG(ERROR) << "Failed to initialize resampler.";
|
||||
return ::mediapipe::UnknownError("Failed to initialize resampler.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TimeSeriesHeader* output_header = new TimeSeriesHeader(input_header);
|
||||
output_header->set_sample_rate(target_sample_rate_);
|
||||
// The resampler doesn't make guarantees about how many samples will
|
||||
// be in each packet.
|
||||
output_header->clear_packet_rate();
|
||||
output_header->clear_num_samples();
|
||||
|
||||
cc->Outputs().Index(0).SetHeader(Adopt(output_header));
|
||||
cumulative_output_samples_ = 0;
|
||||
cumulative_input_samples_ = 0;
|
||||
initial_timestamp_ = Timestamp::Unstarted();
|
||||
check_inconsistent_timestamps_ =
|
||||
resample_options.check_inconsistent_timestamps();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status RationalFactorResampleCalculator::ProcessInternal(
|
||||
const Matrix& input_frame, bool should_flush, CalculatorContext* cc) {
|
||||
if (initial_timestamp_ == Timestamp::Unstarted()) {
|
||||
initial_timestamp_ = cc->InputTimestamp();
|
||||
}
|
||||
|
||||
if (check_inconsistent_timestamps_) {
|
||||
time_series_util::LogWarningIfTimestampIsInconsistent(
|
||||
cc->InputTimestamp(), initial_timestamp_, cumulative_input_samples_,
|
||||
source_sample_rate_);
|
||||
}
|
||||
Timestamp output_timestamp =
|
||||
initial_timestamp_ + ((cumulative_output_samples_ / target_sample_rate_) *
|
||||
Timestamp::kTimestampUnitsPerSecond);
|
||||
|
||||
cumulative_input_samples_ += input_frame.cols();
|
||||
std::unique_ptr<Matrix> output_frame(new Matrix(num_channels_, 0));
|
||||
if (resampler_.empty()) {
|
||||
// Sample rates were same for input and output; pass-thru.
|
||||
*output_frame = input_frame;
|
||||
} else {
|
||||
if (!Resample(input_frame, output_frame.get(), should_flush)) {
|
||||
return ::mediapipe::UnknownError("Resample() failed.");
|
||||
}
|
||||
}
|
||||
cumulative_output_samples_ += output_frame->cols();
|
||||
|
||||
if (output_frame->cols() > 0) {
|
||||
cc->Outputs().Index(0).Add(output_frame.release(), output_timestamp);
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
bool RationalFactorResampleCalculator::Resample(const Matrix& input_frame,
|
||||
Matrix* output_frame,
|
||||
bool should_flush) {
|
||||
std::vector<float> input_vector;
|
||||
std::vector<float> output_vector;
|
||||
for (int i = 0; i < input_frame.rows(); ++i) {
|
||||
CopyChannelToVector(input_frame, i, &input_vector);
|
||||
if (should_flush) {
|
||||
resampler_[i]->Flush(&output_vector);
|
||||
} else {
|
||||
resampler_[i]->ProcessSamples(input_vector, &output_vector);
|
||||
}
|
||||
CopyVectorToChannel(output_vector, output_frame, i);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// static
|
||||
std::unique_ptr<Resampler<float>>
|
||||
RationalFactorResampleCalculator::ResamplerFromOptions(
|
||||
const double source_sample_rate, const double target_sample_rate,
|
||||
const RationalFactorResampleCalculatorOptions& options) {
|
||||
std::unique_ptr<Resampler<float>> resampler;
|
||||
const auto& rational_factor_options =
|
||||
options.resampler_rational_factor_options();
|
||||
std::unique_ptr<DefaultResamplingKernel> kernel;
|
||||
if (rational_factor_options.has_radius() &&
|
||||
rational_factor_options.has_cutoff() &&
|
||||
rational_factor_options.has_kaiser_beta()) {
|
||||
kernel = absl::make_unique<DefaultResamplingKernel>(
|
||||
source_sample_rate, target_sample_rate,
|
||||
rational_factor_options.radius(), rational_factor_options.cutoff(),
|
||||
rational_factor_options.kaiser_beta());
|
||||
} else {
|
||||
kernel = absl::make_unique<DefaultResamplingKernel>(source_sample_rate,
|
||||
target_sample_rate);
|
||||
}
|
||||
|
||||
// Set large enough so that the resampling factor between common sample
|
||||
// rates (e.g. 8kHz, 16kHz, 22.05kHz, 32kHz, 44.1kHz, 48kHz) is exact, and
|
||||
// that any factor is represented with error less than 0.025%.
|
||||
const int kMaxDenominator = 2000;
|
||||
resampler = absl::make_unique<RationalFactorResampler<float>>(
|
||||
*kernel, kMaxDenominator);
|
||||
if (resampler != nullptr && !resampler->Valid()) {
|
||||
resampler = std::unique_ptr<Resampler<float>>();
|
||||
}
|
||||
return resampler;
|
||||
}
|
||||
|
||||
REGISTER_CALCULATOR(RationalFactorResampleCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,106 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_AUDIO_RATIONAL_FACTOR_RESAMPLE_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_AUDIO_RATIONAL_FACTOR_RESAMPLE_CALCULATOR_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "audio/dsp/resampler.h"
|
||||
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/util/time_series_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
// MediaPipe Calculator for resampling a (vector-valued)
|
||||
// input time series with a uniform sample rate. The output
|
||||
// stream's sampling rate is specified by target_sample_rate in the
|
||||
// RationalFactorResampleCalculatorOptions. The output time series may have
|
||||
// a varying number of samples per frame.
|
||||
class RationalFactorResampleCalculator : public CalculatorBase {
|
||||
public:
|
||||
struct TestAccess;
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<Matrix>(
|
||||
// Single input stream with TimeSeriesHeader.
|
||||
);
|
||||
cc->Outputs().Index(0).Set<Matrix>(
|
||||
// Resampled stream with TimeSeriesHeader.
|
||||
);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
// Returns FAIL if the input stream header is invalid or if the
|
||||
// resampler cannot be initialized.
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
// Resamples a packet of TimeSeries data. Returns FAIL if the
|
||||
// resampler state becomes inconsistent.
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
// Flushes any remaining state. Returns FAIL if the resampler state
|
||||
// becomes inconsistent.
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
protected:
|
||||
typedef audio_dsp::Resampler<float> ResamplerType;
|
||||
|
||||
// Returns a Resampler<float> implementation specified by the
|
||||
// RationalFactorResampleCalculatorOptions proto. Returns null if the options
|
||||
// specify an invalid resampler.
|
||||
static std::unique_ptr<ResamplerType> ResamplerFromOptions(
|
||||
const double source_sample_rate, const double target_sample_rate,
|
||||
const RationalFactorResampleCalculatorOptions& options);
|
||||
|
||||
// Does Timestamp bookkeeping and resampling common to Process() and
|
||||
// Close(). Returns FAIL if the resampler state becomes
|
||||
// inconsistent.
|
||||
::mediapipe::Status ProcessInternal(const Matrix& input_frame,
|
||||
bool should_flush, CalculatorContext* cc);
|
||||
|
||||
// Uses the internal resampler_ objects to actually resample each
|
||||
// row of the input TimeSeries. Returns false if the resampler
|
||||
// state becomes inconsistent.
|
||||
bool Resample(const Matrix& input_frame, Matrix* output_frame,
|
||||
bool should_flush);
|
||||
|
||||
double source_sample_rate_;
|
||||
double target_sample_rate_;
|
||||
int64 cumulative_input_samples_;
|
||||
int64 cumulative_output_samples_;
|
||||
Timestamp initial_timestamp_;
|
||||
bool check_inconsistent_timestamps_;
|
||||
int num_channels_;
|
||||
std::vector<std::unique_ptr<ResamplerType>> resampler_;
|
||||
};
|
||||
|
||||
// Test-only access to RationalFactorResampleCalculator methods.
|
||||
struct RationalFactorResampleCalculator::TestAccess {
|
||||
static std::unique_ptr<ResamplerType> ResamplerFromOptions(
|
||||
const double source_sample_rate, const double target_sample_rate,
|
||||
const RationalFactorResampleCalculatorOptions& options) {
|
||||
return RationalFactorResampleCalculator::ResamplerFromOptions(
|
||||
source_sample_rate, target_sample_rate, options);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_AUDIO_RATIONAL_FACTOR_RESAMPLE_CALCULATOR_H_
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message RationalFactorResampleCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional RationalFactorResampleCalculatorOptions ext = 259760074;
|
||||
}
|
||||
|
||||
// target_sample_rate is the sample rate, in Hertz, of the output
|
||||
// stream. Required. Must be greater than 0.
|
||||
optional double target_sample_rate = 1;
|
||||
|
||||
// Parameters for initializing the RationalFactorResampler. See
|
||||
// RationalFactorResampler for more details.
|
||||
message ResamplerRationalFactorOptions {
|
||||
// Kernel radius in units of input samples.
|
||||
optional double radius = 1;
|
||||
// Anti-aliasing cutoff frequency in Hertz. A reasonable setting is
|
||||
// 0.45 * min(input_sample_rate, output_sample_rate).
|
||||
optional double cutoff = 2;
|
||||
// The Kaiser beta parameter for the kernel window.
|
||||
optional double kaiser_beta = 3 [default = 6.0];
|
||||
}
|
||||
optional ResamplerRationalFactorOptions resampler_rational_factor_options = 2;
|
||||
|
||||
// Set to false to disable checks for jitter in timestamp values. Useful with
|
||||
// live audio input.
|
||||
optional bool check_inconsistent_timestamps = 3 [default = true];
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
|
||||
|
||||
#include <math.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "audio/dsp/signal_vector_util.h"
|
||||
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.pb.h"
|
||||
#include "mediapipe/framework//tool/validate_type.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/time_series_test_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
const int kInitialTimestampOffsetMilliseconds = 4;
|
||||
|
||||
class RationalFactorResampleCalculatorTest
|
||||
: public TimeSeriesCalculatorTest<RationalFactorResampleCalculatorOptions> {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
calculator_name_ = "RationalFactorResampleCalculator";
|
||||
input_sample_rate_ = 4000.0;
|
||||
num_input_channels_ = 3;
|
||||
}
|
||||
|
||||
// Expects two vectors whose lengths are almost the same and whose
|
||||
// elements are equal (for indices that are present in both).
|
||||
//
|
||||
// This is useful because the resampler doesn't make precise
|
||||
// guarantees about its output size.
|
||||
void ExpectVectorMostlyFloatEq(const std::vector<float>& expected,
|
||||
const std::vector<float>& actual) {
|
||||
// Lengths should be close, but don't have to be equal.
|
||||
ASSERT_NEAR(expected.size(), actual.size(), 1);
|
||||
for (int i = 0; i < std::min(expected.size(), actual.size()); ++i) {
|
||||
EXPECT_FLOAT_EQ(expected[i], actual[i]) << " where i=" << i << ".";
|
||||
}
|
||||
}
|
||||
|
||||
// Returns a float value with the sample, channel, and timestamp
|
||||
// separated by a few orders of magnitude, for easy parsing by
|
||||
// humans.
|
||||
double TestValue(int sample, int channel, int timestamp_in_microseconds) {
|
||||
return timestamp_in_microseconds * 100.0 + sample + channel / 10.0;
|
||||
}
|
||||
|
||||
// Caller takes ownership of the returned value.
|
||||
Matrix* NewTestFrame(int num_channels, int num_samples, int timestamp) {
|
||||
auto matrix = new Matrix(num_channels, num_samples);
|
||||
for (int c = 0; c < num_channels; ++c) {
|
||||
for (int i = 0; i < num_samples; ++i) {
|
||||
(*matrix)(c, i) = TestValue(i, c, timestamp);
|
||||
}
|
||||
}
|
||||
return matrix;
|
||||
}
|
||||
|
||||
// Initializes and runs the test graph.
|
||||
::mediapipe::Status Run(double output_sample_rate) {
|
||||
options_.set_target_sample_rate(output_sample_rate);
|
||||
InitializeGraph();
|
||||
|
||||
FillInputHeader();
|
||||
concatenated_input_samples_.resize(num_input_channels_, 0);
|
||||
num_input_samples_ = 0;
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
int packet_size = (i + 1) * 10;
|
||||
int timestamp = kInitialTimestampOffsetMilliseconds +
|
||||
num_input_samples_ / input_sample_rate_ *
|
||||
Timestamp::kTimestampUnitsPerSecond;
|
||||
Matrix* data_frame =
|
||||
NewTestFrame(num_input_channels_, packet_size, timestamp);
|
||||
|
||||
// Keep a reference copy of the input.
|
||||
//
|
||||
// conservativeResize() is needed here to preserve the existing
|
||||
// data. Eigen's resize() resizes without preserving data.
|
||||
concatenated_input_samples_.conservativeResize(
|
||||
num_input_channels_, num_input_samples_ + packet_size);
|
||||
concatenated_input_samples_.rightCols(packet_size) = *data_frame;
|
||||
num_input_samples_ += packet_size;
|
||||
|
||||
AppendInputPacket(data_frame, timestamp);
|
||||
}
|
||||
|
||||
return RunGraph();
|
||||
}
|
||||
|
||||
void CheckOutputLength(double output_sample_rate) {
|
||||
double factor = output_sample_rate / input_sample_rate_;
|
||||
|
||||
int num_output_samples = 0;
|
||||
for (const Packet& packet : output().packets) {
|
||||
num_output_samples += packet.Get<Matrix>().cols();
|
||||
}
|
||||
|
||||
// The exact number of expected samples may vary based on the implementation
|
||||
// of the resampler since the exact value is not an integer.
|
||||
// TODO: Reduce this offset to + 1 once cl/185829520 is submitted.
|
||||
const double expected_num_output_samples = num_input_samples_ * factor;
|
||||
EXPECT_LE(ceil(expected_num_output_samples), num_output_samples);
|
||||
EXPECT_GE(ceil(expected_num_output_samples) + 11, num_output_samples);
|
||||
}
|
||||
|
||||
// Checks that output timestamps are consistent with the
|
||||
// output_sample_rate and output packet sizes.
|
||||
void CheckOutputPacketTimestamps(double output_sample_rate) {
|
||||
int num_output_samples = 0;
|
||||
for (const Packet& packet : output().packets) {
|
||||
const int expected_timestamp = kInitialTimestampOffsetMilliseconds +
|
||||
num_output_samples / output_sample_rate *
|
||||
Timestamp::kTimestampUnitsPerSecond;
|
||||
EXPECT_NEAR(expected_timestamp, packet.Timestamp().Value(), 1);
|
||||
num_output_samples += packet.Get<Matrix>().cols();
|
||||
}
|
||||
}
|
||||
|
||||
// Checks that output values from the calculator (which resamples
|
||||
// packet-by-packet) are consistent with resampling the entire
|
||||
// signal at once.
|
||||
void CheckOutputValues(double output_sample_rate) {
|
||||
for (int i = 0; i < num_input_channels_; ++i) {
|
||||
auto verification_resampler =
|
||||
RationalFactorResampleCalculator::TestAccess::ResamplerFromOptions(
|
||||
input_sample_rate_, output_sample_rate, options_);
|
||||
|
||||
std::vector<float> input_data;
|
||||
for (int j = 0; j < num_input_samples_; ++j) {
|
||||
input_data.push_back(concatenated_input_samples_(i, j));
|
||||
}
|
||||
std::vector<float> expected_resampled_data;
|
||||
std::vector<float> temp;
|
||||
verification_resampler->ProcessSamples(input_data, &temp);
|
||||
audio_dsp::VectorAppend(&expected_resampled_data, temp);
|
||||
verification_resampler->Flush(&temp);
|
||||
audio_dsp::VectorAppend(&expected_resampled_data, temp);
|
||||
std::vector<float> actual_resampled_data;
|
||||
for (const Packet& packet : output().packets) {
|
||||
Matrix output_frame_row = packet.Get<Matrix>().row(i);
|
||||
actual_resampled_data.insert(
|
||||
actual_resampled_data.end(), &output_frame_row(0),
|
||||
&output_frame_row(0) + output_frame_row.cols());
|
||||
}
|
||||
|
||||
ExpectVectorMostlyFloatEq(expected_resampled_data, actual_resampled_data);
|
||||
}
|
||||
}
|
||||
|
||||
void CheckOutputHeaders(double output_sample_rate) {
|
||||
const TimeSeriesHeader& output_header =
|
||||
output().header.Get<TimeSeriesHeader>();
|
||||
TimeSeriesHeader expected_header;
|
||||
expected_header.set_sample_rate(output_sample_rate);
|
||||
expected_header.set_num_channels(num_input_channels_);
|
||||
EXPECT_THAT(output_header, mediapipe::EqualsProto(expected_header));
|
||||
}
|
||||
|
||||
void CheckOutput(double output_sample_rate) {
|
||||
CheckOutputLength(output_sample_rate);
|
||||
CheckOutputPacketTimestamps(output_sample_rate);
|
||||
CheckOutputValues(output_sample_rate);
|
||||
CheckOutputHeaders(output_sample_rate);
|
||||
}
|
||||
|
||||
void CheckOutputUnchanged() {
|
||||
for (int i = 0; i < num_input_channels_; ++i) {
|
||||
std::vector<float> expected_resampled_data;
|
||||
for (int j = 0; j < num_input_samples_; ++j) {
|
||||
expected_resampled_data.push_back(concatenated_input_samples_(i, j));
|
||||
}
|
||||
std::vector<float> actual_resampled_data;
|
||||
for (const Packet& packet : output().packets) {
|
||||
Matrix output_frame_row = packet.Get<Matrix>().row(i);
|
||||
actual_resampled_data.insert(
|
||||
actual_resampled_data.end(), &output_frame_row(0),
|
||||
&output_frame_row(0) + output_frame_row.cols());
|
||||
}
|
||||
ExpectVectorMostlyFloatEq(expected_resampled_data, actual_resampled_data);
|
||||
}
|
||||
}
|
||||
|
||||
int num_input_samples_;
|
||||
Matrix concatenated_input_samples_;
|
||||
};
|
||||
|
||||
TEST_F(RationalFactorResampleCalculatorTest, Upsample) {
|
||||
const double kUpsampleRate = input_sample_rate_ * 1.9;
|
||||
MP_ASSERT_OK(Run(kUpsampleRate));
|
||||
CheckOutput(kUpsampleRate);
|
||||
}
|
||||
|
||||
TEST_F(RationalFactorResampleCalculatorTest, Downsample) {
|
||||
const double kDownsampleRate = input_sample_rate_ / 1.9;
|
||||
MP_ASSERT_OK(Run(kDownsampleRate));
|
||||
CheckOutput(kDownsampleRate);
|
||||
}
|
||||
|
||||
TEST_F(RationalFactorResampleCalculatorTest, UsesRationalFactorResampler) {
|
||||
const double kUpsampleRate = input_sample_rate_ * 2;
|
||||
MP_ASSERT_OK(Run(kUpsampleRate));
|
||||
CheckOutput(kUpsampleRate);
|
||||
}
|
||||
|
||||
TEST_F(RationalFactorResampleCalculatorTest, PassthroughIfSampleRateUnchanged) {
|
||||
const double kUpsampleRate = input_sample_rate_;
|
||||
MP_ASSERT_OK(Run(kUpsampleRate));
|
||||
CheckOutputUnchanged();
|
||||
}
|
||||
|
||||
TEST_F(RationalFactorResampleCalculatorTest, FailsOnBadTargetRate) {
|
||||
ASSERT_FALSE(Run(-999.9).ok()); // Invalid output sample rate.
|
||||
}
|
||||
|
||||
TEST_F(RationalFactorResampleCalculatorTest, DoesNotDieOnEmptyInput) {
|
||||
options_.set_target_sample_rate(input_sample_rate_);
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
MP_ASSERT_OK(RunGraph());
|
||||
EXPECT_TRUE(output().packets.empty());
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,436 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Defines SpectrogramCalculator.
|
||||
#include <math.h>
|
||||
|
||||
#include <complex>
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "audio/dsp/spectrogram/spectrogram.h"
|
||||
#include "audio/dsp/window_functions.h"
|
||||
#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/core_proto_inc.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/source_location.h"
|
||||
#include "mediapipe/framework/port/status_builder.h"
|
||||
#include "mediapipe/util/time_series_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// MediaPipe Calculator for computing the "spectrogram" (short-time Fourier
|
||||
// transform squared-magnitude, by default) of a multichannel input
|
||||
// time series, including optionally overlapping frames. Options are
|
||||
// specified in SpectrogramCalculatorOptions proto (where names are chosen
|
||||
// to mirror TimeSeriesFramerCalculator):
|
||||
//
|
||||
// Result is a MatrixData record (for single channel input and when the
|
||||
// allow_multichannel_input flag is false), or a vector of MatrixData records,
|
||||
// one for each channel (when the allow_multichannel_input flag is set). The
|
||||
// rows of each spectrogram matrix correspond to the n_fft/2+1 unique complex
|
||||
// values, or squared/linear/dB magnitudes, depending on the output_type option.
|
||||
// Each input packet will result in zero or one output packets, each containing
|
||||
// one Matrix for each channel of the input, where each Matrix has one or more
|
||||
// columns of spectral values, one for each complete frame of input samples. If
|
||||
// the input packet contains too few samples to trigger a new output frame, no
|
||||
// output packet is generated (since zero-length packets are not legal since
|
||||
// they would result in timestamps that were equal, not strictly increasing).
|
||||
//
|
||||
// Output packet Timestamps are set to the beginning of each frame. This is to
|
||||
// allow calculators downstream from SpectrogramCalculator to have aligned
|
||||
// Timestamps regardless of a packet's signal length.
|
||||
//
|
||||
// Both frame_duration_seconds and frame_overlap_seconds will be
|
||||
// rounded to the nearest integer number of samples. Conseqently, all output
|
||||
// frames will be based on the same number of input samples, and each
|
||||
// analysis frame will advance from its predecessor by the same time step.
|
||||
class SpectrogramCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<Matrix>(
|
||||
// Input stream with TimeSeriesHeader.
|
||||
);
|
||||
|
||||
SpectrogramCalculatorOptions spectrogram_options =
|
||||
cc->Options<SpectrogramCalculatorOptions>();
|
||||
if (!spectrogram_options.allow_multichannel_input()) {
|
||||
if (spectrogram_options.output_type() ==
|
||||
SpectrogramCalculatorOptions::COMPLEX) {
|
||||
cc->Outputs().Index(0).Set<Eigen::MatrixXcf>(
|
||||
// Complex spectrogram frames with TimeSeriesHeader.
|
||||
);
|
||||
} else {
|
||||
cc->Outputs().Index(0).Set<Matrix>(
|
||||
// Spectrogram frames with TimeSeriesHeader.
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if (spectrogram_options.output_type() ==
|
||||
SpectrogramCalculatorOptions::COMPLEX) {
|
||||
cc->Outputs().Index(0).Set<std::vector<Eigen::MatrixXcf>>(
|
||||
// Complex spectrogram frames with MultiStreamTimeSeriesHeader.
|
||||
);
|
||||
} else {
|
||||
cc->Outputs().Index(0).Set<std::vector<Matrix>>(
|
||||
// Spectrogram frames with MultiStreamTimeSeriesHeader.
|
||||
);
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
// Returns FAIL if the input stream header is invalid.
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
|
||||
// Outputs at most one packet consisting of a single Matrix with one or
|
||||
// more columns containing the spectral values from as many input frames
|
||||
// as are completed by the input samples. Always returns OK.
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
// Performs zero-padding and processing of any remaining samples
|
||||
// if pad_final_packet is set.
|
||||
// Returns OK.
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
Timestamp CurrentOutputTimestamp(CalculatorContext* cc) {
|
||||
if (use_local_timestamp_) {
|
||||
return cc->InputTimestamp();
|
||||
}
|
||||
return CumulativeOutputTimestamp();
|
||||
}
|
||||
|
||||
Timestamp CumulativeOutputTimestamp() {
|
||||
// Cumulative output timestamp is the *center* of the next frame to be
|
||||
// emitted, hence delayed by half a window duration compared to relevant
|
||||
// input timestamp.
|
||||
return initial_input_timestamp_ +
|
||||
round(cumulative_completed_frames_ * frame_step_samples() *
|
||||
Timestamp::kTimestampUnitsPerSecond / input_sample_rate_);
|
||||
}
|
||||
|
||||
int frame_step_samples() const {
|
||||
return frame_duration_samples_ - frame_overlap_samples_;
|
||||
}
|
||||
|
||||
// Take the next set of input samples, already translated into a
|
||||
// vector<float> and pass them to the spectrogram object.
|
||||
// Convert the output of the spectrogram object into a Matrix (or an
|
||||
// Eigen::MatrixXcf if complex-valued output is requested) and pass to
|
||||
// MediaPipe output.
|
||||
::mediapipe::Status ProcessVector(const Matrix& input_stream,
|
||||
CalculatorContext* cc);
|
||||
|
||||
// Templated function to process either real- or complex-output spectrogram.
|
||||
template <class OutputMatrixType>
|
||||
::mediapipe::Status ProcessVectorToOutput(
|
||||
const Matrix& input_stream,
|
||||
const OutputMatrixType postprocess_output_fn(const OutputMatrixType&),
|
||||
CalculatorContext* cc);
|
||||
|
||||
bool use_local_timestamp_;
|
||||
double input_sample_rate_;
|
||||
bool pad_final_packet_;
|
||||
int frame_duration_samples_;
|
||||
int frame_overlap_samples_;
|
||||
// How many samples we've been passed, used for checking input time stamps.
|
||||
int64 cumulative_input_samples_;
|
||||
// How many frames we've emitted, used for calculating output time stamps.
|
||||
int64 cumulative_completed_frames_;
|
||||
Timestamp initial_input_timestamp_;
|
||||
int num_input_channels_;
|
||||
// How many frequency bins we emit (=N_FFT/2 + 1).
|
||||
int num_output_channels_;
|
||||
// Which output type?
|
||||
int output_type_;
|
||||
// Output type: mono or multichannel.
|
||||
bool allow_multichannel_input_;
|
||||
// Vector of Spectrogram objects, one for each channel.
|
||||
std::vector<std::unique_ptr<audio_dsp::Spectrogram>> spectrogram_generators_;
|
||||
// Fixed scale factor applied to output values (regardless of type).
|
||||
double output_scale_;
|
||||
|
||||
static const float kLnPowerToDb;
|
||||
};
|
||||
REGISTER_CALCULATOR(SpectrogramCalculator);
|
||||
|
||||
// Factor to convert ln(magnitude_squared) to deciBels = 10.0/ln(10.0).
|
||||
const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
|
||||
|
||||
::mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
|
||||
SpectrogramCalculatorOptions spectrogram_options =
|
||||
cc->Options<SpectrogramCalculatorOptions>();
|
||||
|
||||
use_local_timestamp_ = spectrogram_options.use_local_timestamp();
|
||||
|
||||
if (spectrogram_options.frame_duration_seconds() <= 0.0) {
|
||||
::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "Invalid or missing frame_duration_seconds.\n"
|
||||
"frame_duration_seconds: "
|
||||
<< spectrogram_options.frame_overlap_seconds();
|
||||
}
|
||||
if (spectrogram_options.frame_overlap_seconds() >=
|
||||
spectrogram_options.frame_duration_seconds()) {
|
||||
::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "Invalid frame_overlap_seconds.\nframe_overlap_seconds: "
|
||||
<< spectrogram_options.frame_overlap_seconds()
|
||||
<< "\nframe_duration_seconds: "
|
||||
<< spectrogram_options.frame_duration_seconds();
|
||||
}
|
||||
if (spectrogram_options.frame_overlap_seconds() < 0.0) {
|
||||
::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "Frame_overlap_seconds is < 0.0.\nframe_overlap_seconds: "
|
||||
<< spectrogram_options.frame_overlap_seconds();
|
||||
}
|
||||
|
||||
TimeSeriesHeader input_header;
|
||||
MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
|
||||
cc->Inputs().Index(0).Header(), &input_header));
|
||||
|
||||
input_sample_rate_ = input_header.sample_rate();
|
||||
num_input_channels_ = input_header.num_channels();
|
||||
|
||||
if (!spectrogram_options.allow_multichannel_input() &&
|
||||
num_input_channels_ != 1) {
|
||||
::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "The current setting only supports single-channel input. Please set "
|
||||
"allow_multichannel_input.\n";
|
||||
}
|
||||
|
||||
frame_duration_samples_ =
|
||||
round(spectrogram_options.frame_duration_seconds() * input_sample_rate_);
|
||||
frame_overlap_samples_ =
|
||||
round(spectrogram_options.frame_overlap_seconds() * input_sample_rate_);
|
||||
|
||||
pad_final_packet_ = spectrogram_options.pad_final_packet();
|
||||
output_type_ = spectrogram_options.output_type();
|
||||
allow_multichannel_input_ = spectrogram_options.allow_multichannel_input();
|
||||
|
||||
output_scale_ = spectrogram_options.output_scale();
|
||||
|
||||
std::vector<double> window;
|
||||
switch (spectrogram_options.window_type()) {
|
||||
case SpectrogramCalculatorOptions::COSINE:
|
||||
audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
|
||||
&window);
|
||||
break;
|
||||
case SpectrogramCalculatorOptions::HANN:
|
||||
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
|
||||
&window);
|
||||
break;
|
||||
case SpectrogramCalculatorOptions::HAMMING:
|
||||
audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
|
||||
&window);
|
||||
break;
|
||||
}
|
||||
|
||||
// Propagate settings down to the actual Spectrogram object.
|
||||
spectrogram_generators_.clear();
|
||||
for (int i = 0; i < num_input_channels_; i++) {
|
||||
spectrogram_generators_.push_back(
|
||||
std::unique_ptr<audio_dsp::Spectrogram>(new audio_dsp::Spectrogram()));
|
||||
spectrogram_generators_[i]->Initialize(window, frame_step_samples());
|
||||
}
|
||||
|
||||
num_output_channels_ =
|
||||
spectrogram_generators_[0]->output_frequency_channels();
|
||||
std::unique_ptr<TimeSeriesHeader> output_header(
|
||||
new TimeSeriesHeader(input_header));
|
||||
// Store the actual sample rate of the input audio in the TimeSeriesHeader
|
||||
// so that subsequent calculators can figure out the frequency scale of
|
||||
// our output.
|
||||
output_header->set_audio_sample_rate(input_sample_rate_);
|
||||
// Setup rest of output header.
|
||||
output_header->set_num_channels(num_output_channels_);
|
||||
output_header->set_sample_rate(input_sample_rate_ / frame_step_samples());
|
||||
// Although we usually generate one output packet for each input
|
||||
// packet, this might not be true for input packets whose size is smaller
|
||||
// than the analysis window length. So we clear output_header.packet_rate
|
||||
// because we can't guarantee a constant packet rate. Similarly, the number
|
||||
// of output frames per packet depends on the input packet, so we also clear
|
||||
// output_header.num_samples.
|
||||
output_header->clear_packet_rate();
|
||||
output_header->clear_num_samples();
|
||||
if (!spectrogram_options.allow_multichannel_input()) {
|
||||
cc->Outputs().Index(0).SetHeader(Adopt(output_header.release()));
|
||||
} else {
|
||||
std::unique_ptr<MultiStreamTimeSeriesHeader> multichannel_output_header(
|
||||
new MultiStreamTimeSeriesHeader());
|
||||
*multichannel_output_header->mutable_time_series_header() = *output_header;
|
||||
multichannel_output_header->set_num_streams(num_input_channels_);
|
||||
cc->Outputs().Index(0).SetHeader(
|
||||
Adopt(multichannel_output_header.release()));
|
||||
}
|
||||
cumulative_completed_frames_ = 0;
|
||||
initial_input_timestamp_ = Timestamp::Unstarted();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status SpectrogramCalculator::Process(CalculatorContext* cc) {
|
||||
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
|
||||
initial_input_timestamp_ = cc->InputTimestamp();
|
||||
}
|
||||
|
||||
const Matrix& input_stream = cc->Inputs().Index(0).Get<Matrix>();
|
||||
if (input_stream.rows() != num_input_channels_) {
|
||||
::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "Number of input channels do not correspond to the number of rows "
|
||||
<< "in the input matrix: " << num_input_channels_ << "channels vs "
|
||||
<< input_stream.rows() << " rows";
|
||||
}
|
||||
|
||||
cumulative_input_samples_ += input_stream.cols();
|
||||
|
||||
return ProcessVector(input_stream, cc);
|
||||
}
|
||||
|
||||
template <class OutputMatrixType>
|
||||
::mediapipe::Status SpectrogramCalculator::ProcessVectorToOutput(
|
||||
const Matrix& input_stream,
|
||||
const OutputMatrixType postprocess_output_fn(const OutputMatrixType&),
|
||||
CalculatorContext* cc) {
|
||||
std::unique_ptr<std::vector<OutputMatrixType>> spectrogram_matrices(
|
||||
new std::vector<OutputMatrixType>());
|
||||
std::vector<std::vector<typename OutputMatrixType::Scalar>> output_vectors;
|
||||
|
||||
// Compute a spectrogram for each channel.
|
||||
int num_output_time_frames;
|
||||
for (int channel = 0; channel < input_stream.rows(); ++channel) {
|
||||
output_vectors.clear();
|
||||
|
||||
// Copy one row (channel) of the input matrix into the std::vector.
|
||||
std::vector<float> input_vector(input_stream.cols());
|
||||
Eigen::Map<Matrix>(&input_vector[0], 1, input_vector.size()) =
|
||||
input_stream.row(channel);
|
||||
|
||||
if (!spectrogram_generators_[channel]->ComputeSpectrogram(
|
||||
input_vector, &output_vectors)) {
|
||||
return ::mediapipe::Status(mediapipe::StatusCode::kInternal,
|
||||
"Spectrogram returned failure");
|
||||
}
|
||||
if (channel == 0) {
|
||||
// Record the number of time frames we expect from each channel.
|
||||
num_output_time_frames = output_vectors.size();
|
||||
} else {
|
||||
RET_CHECK_EQ(output_vectors.size(), num_output_time_frames)
|
||||
<< "Inconsistent spectrogram time frames for channel " << channel;
|
||||
}
|
||||
// Skip remaining processing if there are too few input samples to trigger
|
||||
// any output frames.
|
||||
if (!output_vectors.empty()) {
|
||||
// Translate the returned values into a matrix of output frames.
|
||||
OutputMatrixType output_frames(num_output_channels_,
|
||||
output_vectors.size());
|
||||
for (int frame = 0; frame < output_vectors.size(); ++frame) {
|
||||
Eigen::Map<const OutputMatrixType> frame_map(
|
||||
&output_vectors[frame][0], output_vectors[frame].size(), 1);
|
||||
// The underlying dsp object returns squared magnitudes; here
|
||||
// we optionally translate to linear magnitude or dB.
|
||||
output_frames.col(frame) =
|
||||
output_scale_ * postprocess_output_fn(frame_map);
|
||||
}
|
||||
spectrogram_matrices->push_back(output_frames);
|
||||
}
|
||||
}
|
||||
// If the input is very short, there may not be enough accumulated,
|
||||
// unprocessed samples to cause any new frames to be generated by
|
||||
// the spectrogram object. If so, we don't want to emit
|
||||
// a packet at all.
|
||||
if (!spectrogram_matrices->empty()) {
|
||||
RET_CHECK_EQ(spectrogram_matrices->size(), input_stream.rows())
|
||||
<< "Inconsistent number of spectrogram channels.";
|
||||
if (allow_multichannel_input_) {
|
||||
cc->Outputs().Index(0).Add(spectrogram_matrices.release(),
|
||||
CurrentOutputTimestamp(cc));
|
||||
} else {
|
||||
cc->Outputs().Index(0).Add(
|
||||
new OutputMatrixType(spectrogram_matrices->at(0)),
|
||||
CurrentOutputTimestamp(cc));
|
||||
}
|
||||
cumulative_completed_frames_ += output_vectors.size();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status SpectrogramCalculator::ProcessVector(
|
||||
const Matrix& input_stream, CalculatorContext* cc) {
|
||||
switch (output_type_) {
|
||||
// These blocks deliberately ignore clang-format to preserve the
|
||||
// "silhouette" of the different cases.
|
||||
// clang-format off
|
||||
case SpectrogramCalculatorOptions::COMPLEX: {
|
||||
return ProcessVectorToOutput(
|
||||
input_stream,
|
||||
+[](const Eigen::MatrixXcf& col) -> const Eigen::MatrixXcf {
|
||||
return col;
|
||||
}, cc);
|
||||
}
|
||||
case SpectrogramCalculatorOptions::SQUARED_MAGNITUDE: {
|
||||
return ProcessVectorToOutput(
|
||||
input_stream,
|
||||
+[](const Matrix& col) -> const Matrix {
|
||||
return col;
|
||||
}, cc);
|
||||
}
|
||||
case SpectrogramCalculatorOptions::LINEAR_MAGNITUDE: {
|
||||
return ProcessVectorToOutput(
|
||||
input_stream,
|
||||
+[](const Matrix& col) -> const Matrix {
|
||||
return col.array().sqrt().matrix();
|
||||
}, cc);
|
||||
}
|
||||
case SpectrogramCalculatorOptions::DECIBELS: {
|
||||
return ProcessVectorToOutput(
|
||||
input_stream,
|
||||
+[](const Matrix& col) -> const Matrix {
|
||||
return kLnPowerToDb * col.array().log().matrix();
|
||||
}, cc);
|
||||
}
|
||||
// clang-format on
|
||||
default: {
|
||||
return ::mediapipe::Status(mediapipe::StatusCode::kInvalidArgument,
|
||||
"Unrecognized spectrogram output type.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
::mediapipe::Status SpectrogramCalculator::Close(CalculatorContext* cc) {
|
||||
if (cumulative_input_samples_ > 0 && pad_final_packet_) {
|
||||
// We can flush any remaining samples by sending frame_step_samples - 1
|
||||
// zeros to the Process method, and letting it do its thing,
|
||||
// UNLESS we have fewer than one window's worth of samples, in which case
|
||||
// we pad to exactly one frame_duration_samples.
|
||||
// Release the memory for the Spectrogram objects.
|
||||
int required_padding_samples = frame_step_samples() - 1;
|
||||
if (cumulative_input_samples_ < frame_duration_samples_) {
|
||||
required_padding_samples =
|
||||
frame_duration_samples_ - cumulative_input_samples_;
|
||||
}
|
||||
return ProcessVector(
|
||||
Matrix::Zero(num_input_channels_, required_padding_samples), cc);
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message SpectrogramCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional SpectrogramCalculatorOptions ext = 76186688;
|
||||
}
|
||||
|
||||
// Options mirror those of TimeSeriesFramerCalculator.
|
||||
|
||||
// Analysis window duration in seconds. Required. Must be greater than 0.
|
||||
// (Note: the spectrogram DFT length will be the smallest power-of-2
|
||||
// sample count that can hold this duration.)
|
||||
optional double frame_duration_seconds = 1;
|
||||
|
||||
// Duration of overlap between adjacent windows.
|
||||
// Hence, frame_rate = 1/(frame_duration_seconds - frame_overlap_seconds).
|
||||
// Required that 0 <= frame_overlap_seconds < frame_duration_seconds.
|
||||
optional double frame_overlap_seconds = 2 [default = 0.0];
|
||||
|
||||
// Whether to pad the final packet with zeros. If true, guarantees that
|
||||
// all input samples will output. If set to false, any partial packet
|
||||
// at the end of the stream will be dropped.
|
||||
optional bool pad_final_packet = 3 [default = true];
|
||||
|
||||
// Output value type can be squared-magnitude, linear-magnitude,
|
||||
// deciBels (dB, = 20*log10(linear_magnitude)), or std::complex.
|
||||
enum OutputType {
|
||||
SQUARED_MAGNITUDE = 0;
|
||||
LINEAR_MAGNITUDE = 1;
|
||||
DECIBELS = 2;
|
||||
COMPLEX = 3;
|
||||
}
|
||||
optional OutputType output_type = 4 [default = SQUARED_MAGNITUDE];
|
||||
|
||||
// If set to true then the output will be a vector of spectrograms, one for
|
||||
// each channel and the stream will have a MultiStreamTimeSeriesHeader.
|
||||
optional bool allow_multichannel_input = 5 [default = false];
|
||||
|
||||
// Which window to use when computing the FFT.
|
||||
enum WindowType {
|
||||
HANN = 0;
|
||||
HAMMING = 1;
|
||||
COSINE = 2;
|
||||
}
|
||||
optional WindowType window_type = 6 [default = HANN];
|
||||
|
||||
// Support a fixed multiplicative scaling of the output. This is applied
|
||||
// uniformly regardless of output type (i.e., even dBs are multiplied, not
|
||||
// offset).
|
||||
optional double output_scale = 7 [default = 1.0];
|
||||
|
||||
// If use_local_timestamp is true, the output packet's timestamp is based on
|
||||
// the last sample of the packet and it's inferred from the latest input
|
||||
// packet's timestamp. If false, the output packet's timestamp is based on
|
||||
// the cumulative timestamping, which is inferred from the intial input
|
||||
// timestamp and the cumulative number of samples.
|
||||
optional bool use_local_timestamp = 8 [default = false];
|
||||
}
|
||||
@@ -0,0 +1,895 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <math.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <complex>
|
||||
#include <memory>
|
||||
#include <numeric>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "audio/dsp/number_util.h"
|
||||
#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/benchmark.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/time_series_test_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
const int kInitialTimestampOffsetMicroseconds = 4;
|
||||
|
||||
class SpectrogramCalculatorTest
|
||||
: public TimeSeriesCalculatorTest<SpectrogramCalculatorOptions> {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
calculator_name_ = "SpectrogramCalculator";
|
||||
input_sample_rate_ = 4000.0;
|
||||
num_input_channels_ = 1;
|
||||
}
|
||||
|
||||
// Initializes and runs the test graph.
|
||||
::mediapipe::Status Run() {
|
||||
// Now that options are set, we can set up some internal constants.
|
||||
frame_duration_samples_ =
|
||||
round(options_.frame_duration_seconds() * input_sample_rate_);
|
||||
frame_step_samples_ =
|
||||
frame_duration_samples_ -
|
||||
round(options_.frame_overlap_seconds() * input_sample_rate_);
|
||||
// The magnitude of the 0th FFT bin (DC) should be sum(input.*window);
|
||||
// for an input identically 1.0, this is just sum(window). The average
|
||||
// value of our Hann window is 0.5, hence this is the expected squared-
|
||||
// magnitude output value in the DC bin for constant input of 1.0.
|
||||
expected_dc_squared_magnitude_ =
|
||||
pow((static_cast<float>(frame_duration_samples_) * 0.5), 2.0);
|
||||
|
||||
return RunGraph();
|
||||
}
|
||||
|
||||
// Creates test multichannel input with specified packet sizes and containing
|
||||
// a constant-frequency sinusoid that maintains phase between adjacent
|
||||
// packets.
|
||||
void SetupCosineInputPackets(const std::vector<int>& packet_sizes_samples,
|
||||
float cosine_frequency_hz) {
|
||||
int total_num_input_samples = 0;
|
||||
for (int packet_size_samples : packet_sizes_samples) {
|
||||
double packet_start_time_seconds =
|
||||
kInitialTimestampOffsetMicroseconds * 1e-6 +
|
||||
total_num_input_samples / input_sample_rate_;
|
||||
double packet_end_time_seconds =
|
||||
packet_start_time_seconds + packet_size_samples / input_sample_rate_;
|
||||
double angular_freq = 2 * M_PI * cosine_frequency_hz;
|
||||
Matrix* packet_data =
|
||||
new Matrix(num_input_channels_, packet_size_samples);
|
||||
// Use Eigen's vectorized cos() function to fill the vector with a
|
||||
// sinusoid of appropriate frequency & phase.
|
||||
for (int i = 0; i < num_input_channels_; i++) {
|
||||
packet_data->row(i) =
|
||||
Eigen::ArrayXf::LinSpaced(packet_size_samples,
|
||||
packet_start_time_seconds * angular_freq,
|
||||
packet_end_time_seconds * angular_freq)
|
||||
.cos()
|
||||
.transpose();
|
||||
}
|
||||
int64 input_timestamp = round(packet_start_time_seconds *
|
||||
Timestamp::kTimestampUnitsPerSecond);
|
||||
AppendInputPacket(packet_data, input_timestamp);
|
||||
total_num_input_samples += packet_size_samples;
|
||||
}
|
||||
}
|
||||
|
||||
// Setup a sequence of input packets of specified sizes, each filled
|
||||
// with samples of 1.0.
|
||||
void SetupConstantInputPackets(const std::vector<int>& packet_sizes_samples) {
|
||||
// A 0 Hz cosine is identically 1.0 for all samples.
|
||||
SetupCosineInputPackets(packet_sizes_samples, 0.0);
|
||||
}
|
||||
|
||||
// Setup a sequence of input packets of specified sizes, each containing a
|
||||
// single sample of 1.0 at a specified offset.
|
||||
void SetupImpulseInputPackets(
|
||||
const std::vector<int>& packet_sizes_samples,
|
||||
const std::vector<int>& impulse_offsets_samples) {
|
||||
int total_num_input_samples = 0;
|
||||
for (int i = 0; i < packet_sizes_samples.size(); ++i) {
|
||||
double packet_start_time_seconds =
|
||||
kInitialTimestampOffsetMicroseconds * 1e-6 +
|
||||
total_num_input_samples / input_sample_rate_;
|
||||
int64 input_timestamp = round(packet_start_time_seconds *
|
||||
Timestamp::kTimestampUnitsPerSecond);
|
||||
std::unique_ptr<Matrix> impulse(
|
||||
new Matrix(Matrix::Zero(1, packet_sizes_samples[i])));
|
||||
(*impulse)(0, impulse_offsets_samples[i]) = 1.0;
|
||||
AppendInputPacket(impulse.release(), input_timestamp);
|
||||
total_num_input_samples += packet_sizes_samples[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Creates test multichannel input with specified packet sizes and containing
|
||||
// constant input packets for the even channels and constant-frequency
|
||||
// sinusoid that maintains phase between adjacent packets for the odd
|
||||
// channels.
|
||||
void SetupMultichannelInputPackets(
|
||||
const std::vector<int>& packet_sizes_samples, float cosine_frequency_hz) {
|
||||
int total_num_input_samples = 0;
|
||||
for (int packet_size_samples : packet_sizes_samples) {
|
||||
double packet_start_time_seconds =
|
||||
kInitialTimestampOffsetMicroseconds * 1e-6 +
|
||||
total_num_input_samples / input_sample_rate_;
|
||||
double packet_end_time_seconds =
|
||||
packet_start_time_seconds + packet_size_samples / input_sample_rate_;
|
||||
double angular_freq;
|
||||
Matrix* packet_data =
|
||||
new Matrix(num_input_channels_, packet_size_samples);
|
||||
// Use Eigen's vectorized cos() function to fill the vector with a
|
||||
// sinusoid of appropriate frequency & phase.
|
||||
for (int i = 0; i < num_input_channels_; i++) {
|
||||
if (i % 2 == 0) {
|
||||
angular_freq = 0;
|
||||
} else {
|
||||
angular_freq = 2 * M_PI * cosine_frequency_hz;
|
||||
}
|
||||
packet_data->row(i) =
|
||||
Eigen::ArrayXf::LinSpaced(packet_size_samples,
|
||||
packet_start_time_seconds * angular_freq,
|
||||
packet_end_time_seconds * angular_freq)
|
||||
.cos()
|
||||
.transpose();
|
||||
}
|
||||
int64 input_timestamp = round(packet_start_time_seconds *
|
||||
Timestamp::kTimestampUnitsPerSecond);
|
||||
AppendInputPacket(packet_data, input_timestamp);
|
||||
total_num_input_samples += packet_size_samples;
|
||||
}
|
||||
}
|
||||
|
||||
// Return vector of the numbers of frames in each output packet.
|
||||
std::vector<int> OutputFramesPerPacket() {
|
||||
std::vector<int> frame_counts;
|
||||
for (const Packet& packet : output().packets) {
|
||||
const Matrix& matrix = packet.Get<Matrix>();
|
||||
frame_counts.push_back(matrix.cols());
|
||||
}
|
||||
return frame_counts;
|
||||
}
|
||||
|
||||
// Checks output headers and Timestamps.
|
||||
void CheckOutputHeadersAndTimestamps() {
|
||||
const int fft_size = audio_dsp::NextPowerOfTwo(frame_duration_samples_);
|
||||
|
||||
TimeSeriesHeader expected_header = input().header.Get<TimeSeriesHeader>();
|
||||
expected_header.set_num_channels(fft_size / 2 + 1);
|
||||
// The output header sample rate should depend on the output frame step.
|
||||
expected_header.set_sample_rate(input_sample_rate_ / frame_step_samples_);
|
||||
// SpectrogramCalculator stores the sample rate of the input in
|
||||
// the TimeSeriesHeader.
|
||||
expected_header.set_audio_sample_rate(input_sample_rate_);
|
||||
// We expect the output header to have num_samples and packet_rate unset.
|
||||
expected_header.clear_num_samples();
|
||||
expected_header.clear_packet_rate();
|
||||
if (!options_.allow_multichannel_input()) {
|
||||
ExpectOutputHeaderEquals(expected_header);
|
||||
} else {
|
||||
EXPECT_THAT(output()
|
||||
.header.template Get<MultiStreamTimeSeriesHeader>()
|
||||
.time_series_header(),
|
||||
mediapipe::EqualsProto(expected_header));
|
||||
EXPECT_THAT(output()
|
||||
.header.template Get<MultiStreamTimeSeriesHeader>()
|
||||
.num_streams(),
|
||||
num_input_channels_);
|
||||
}
|
||||
|
||||
int cumulative_output_frames = 0;
|
||||
// The timestamps coming out of the spectrogram correspond to the
|
||||
// middle of the first frame's window, hence frame_duration_samples_/2
|
||||
// term. We use frame_duration_samples_ because that is how it is
|
||||
// actually quantized inside spectrogram.
|
||||
const double packet_timestamp_offset_seconds =
|
||||
kInitialTimestampOffsetMicroseconds * 1e-6;
|
||||
const double frame_step_seconds = frame_step_samples_ / input_sample_rate_;
|
||||
|
||||
Timestamp initial_timestamp = Timestamp::Unstarted();
|
||||
|
||||
for (const Packet& packet : output().packets) {
|
||||
// This is the timestamp we expect based on how the spectrogram should
|
||||
// behave (advancing by one step's worth of input samples each frame).
|
||||
const double expected_timestamp_seconds =
|
||||
packet_timestamp_offset_seconds +
|
||||
cumulative_output_frames * frame_step_seconds;
|
||||
const int64 expected_timestamp_ticks =
|
||||
expected_timestamp_seconds * Timestamp::kTimestampUnitsPerSecond;
|
||||
EXPECT_EQ(expected_timestamp_ticks, packet.Timestamp().Value());
|
||||
// Accept the timestamp of the first packet as the baseline for checking
|
||||
// the remainder.
|
||||
if (initial_timestamp == Timestamp::Unstarted()) {
|
||||
initial_timestamp = packet.Timestamp();
|
||||
}
|
||||
// Also check that the timestamp is consistent with the sample_rate
|
||||
// in the output stream's TimeSeriesHeader.
|
||||
EXPECT_TRUE(time_series_util::LogWarningIfTimestampIsInconsistent(
|
||||
packet.Timestamp(), initial_timestamp, cumulative_output_frames,
|
||||
expected_header.sample_rate()));
|
||||
if (!options_.allow_multichannel_input()) {
|
||||
if (options_.output_type() == SpectrogramCalculatorOptions::COMPLEX) {
|
||||
const Eigen::MatrixXcf& matrix = packet.Get<Eigen::MatrixXcf>();
|
||||
cumulative_output_frames += matrix.cols();
|
||||
} else {
|
||||
const Matrix& matrix = packet.Get<Matrix>();
|
||||
cumulative_output_frames += matrix.cols();
|
||||
}
|
||||
} else {
|
||||
if (options_.output_type() == SpectrogramCalculatorOptions::COMPLEX) {
|
||||
const Eigen::MatrixXcf& matrix =
|
||||
packet.Get<std::vector<Eigen::MatrixXcf>>().at(0);
|
||||
cumulative_output_frames += matrix.cols();
|
||||
} else {
|
||||
const Matrix& matrix = packet.Get<std::vector<Matrix>>().at(0);
|
||||
cumulative_output_frames += matrix.cols();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify that the bin corresponding to the specified frequency
|
||||
// is the largest one in one particular frame of a single packet.
|
||||
void CheckPeakFrequencyInPacketFrame(const Packet& packet, int frame,
|
||||
float frequency) {
|
||||
const int fft_size = audio_dsp::NextPowerOfTwo(frame_duration_samples_);
|
||||
const int target_bin =
|
||||
round((frequency / input_sample_rate_) * static_cast<float>(fft_size));
|
||||
|
||||
const Matrix& matrix = packet.Get<Matrix>();
|
||||
// Stop here if the requested frame is not in this packet.
|
||||
ASSERT_GT(matrix.cols(), frame);
|
||||
|
||||
int actual_largest_bin;
|
||||
matrix.col(frame).maxCoeff(&actual_largest_bin);
|
||||
EXPECT_EQ(actual_largest_bin, target_bin);
|
||||
}
|
||||
|
||||
// Verify that the bin corresponding to the specified frequency
|
||||
// is the largest one in one particular frame of a single spectrogram Matrix.
|
||||
void CheckPeakFrequencyInMatrix(const Matrix& matrix, int frame,
|
||||
float frequency) {
|
||||
const int fft_size = audio_dsp::NextPowerOfTwo(frame_duration_samples_);
|
||||
const int target_bin =
|
||||
round((frequency / input_sample_rate_) * static_cast<float>(fft_size));
|
||||
|
||||
// Stop here if the requested frame is not in this packet.
|
||||
ASSERT_GT(matrix.cols(), frame);
|
||||
|
||||
int actual_largest_bin;
|
||||
matrix.col(frame).maxCoeff(&actual_largest_bin);
|
||||
EXPECT_EQ(actual_largest_bin, target_bin);
|
||||
}
|
||||
|
||||
int frame_duration_samples_;
|
||||
int frame_step_samples_;
|
||||
// Expected DC output for a window of pure 1.0, set when window length
|
||||
// is set.
|
||||
float expected_dc_squared_magnitude_;
|
||||
};
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, IntegerFrameDurationNoOverlap) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(0.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
const std::vector<int> input_packet_sizes = {500, 200};
|
||||
const std::vector<int> expected_output_packet_sizes = {5, 2};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, IntegerFrameDurationSomeOverlap) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
const std::vector<int> input_packet_sizes = {500, 200};
|
||||
// complete_output_frames = 1 + floor((input_samples - window_length)/step)
|
||||
// = 1 + floor((500 - 100)/40) = 1 + 10 = 11 for the first packet
|
||||
// = 1 + floor((700 - 100)/40) = 1 + 15 = 16 for the whole stream
|
||||
// so expect 16 - 11 = 5 in the second packet.
|
||||
const std::vector<int> expected_output_packet_sizes = {11, 5};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, NonintegerFrameDurationAndOverlap) {
|
||||
options_.set_frame_duration_seconds(98.5 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(58.4 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
const std::vector<int> input_packet_sizes = {500, 200};
|
||||
// now frame_duration_samples will be 99 (rounded), and frame_step_samples
|
||||
// will be (99-58) = 41, so the first packet of 500 samples will generate
|
||||
// 1 + floor(500-99)/41 = 10 samples.
|
||||
const std::vector<int> expected_output_packet_sizes = {10, 5};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, ShortInitialPacketNoOverlap) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(0.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
const std::vector<int> input_packet_sizes = {90, 100, 110};
|
||||
// The first input packet is too small to generate any frames,
|
||||
// but zero-length packets would result in a timestamp monotonicity
|
||||
// violation, so they are suppressed. Thus, only the second and third
|
||||
// input packets generate output packets.
|
||||
const std::vector<int> expected_output_packet_sizes = {1, 2};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, TrailingSamplesNoPad) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
const std::vector<int> input_packet_sizes = {140, 90};
|
||||
const std::vector<int> expected_output_packet_sizes = {2, 2};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, NoTrailingSamplesWithPad) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(true);
|
||||
const std::vector<int> input_packet_sizes = {140, 80};
|
||||
const std::vector<int> expected_output_packet_sizes = {2, 2};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, TrailingSamplesWithPad) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(true);
|
||||
const std::vector<int> input_packet_sizes = {140, 90};
|
||||
// In contrast to NoTrailingSamplesWithPad and TrailingSamplesNoPad,
|
||||
// this time we get an extra frame in an extra final packet.
|
||||
const std::vector<int> expected_output_packet_sizes = {2, 2, 1};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, VeryShortInputWillPad) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(true);
|
||||
const std::vector<int> input_packet_sizes = {30};
|
||||
const std::vector<int> expected_output_packet_sizes = {1};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, VeryShortInputZeroOutputFramesIfNoPad) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
const std::vector<int> input_packet_sizes = {90};
|
||||
const std::vector<int> expected_output_packet_sizes = {};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, DCSignalIsPeakBin) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
const std::vector<int> input_packet_sizes = {140}; // Gives 2 output frames.
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
// Setup packets with DC input (non-zero constant value).
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
const float dc_frequency_hz = 0.0;
|
||||
CheckPeakFrequencyInPacketFrame(output().packets[0], 0, dc_frequency_hz);
|
||||
CheckPeakFrequencyInPacketFrame(output().packets[0], 1, dc_frequency_hz);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, A440ToneIsPeakBin) {
|
||||
const std::vector<int> input_packet_sizes = {
|
||||
460}; // 100 + 9*40 for 10 frames.
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
const float tone_frequency_hz = 440.0;
|
||||
SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
int num_output_frames = output().packets[0].Get<Matrix>().cols();
|
||||
for (int frame = 0; frame < num_output_frames; ++frame) {
|
||||
CheckPeakFrequencyInPacketFrame(output().packets[0], frame,
|
||||
tone_frequency_hz);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, SquaredMagnitudeOutputLooksRight) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_output_type(SpectrogramCalculatorOptions::SQUARED_MAGNITUDE);
|
||||
const std::vector<int> input_packet_sizes = {140};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
// Setup packets with DC input (non-zero constant value).
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
|
||||
expected_dc_squared_magnitude_);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, DefaultOutputIsSquaredMagnitude) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
// Let the output_type be its default
|
||||
const std::vector<int> input_packet_sizes = {140};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
// Setup packets with DC input (non-zero constant value).
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
|
||||
expected_dc_squared_magnitude_);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, LinearMagnitudeOutputLooksRight) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_output_type(SpectrogramCalculatorOptions::LINEAR_MAGNITUDE);
|
||||
const std::vector<int> input_packet_sizes = {140};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
// Setup packets with DC input (non-zero constant value).
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
|
||||
std::sqrt(expected_dc_squared_magnitude_));
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, DbMagnitudeOutputLooksRight) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_output_type(SpectrogramCalculatorOptions::DECIBELS);
|
||||
const std::vector<int> input_packet_sizes = {140};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
// Setup packets with DC input (non-zero constant value).
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
|
||||
10.0 * std::log10(expected_dc_squared_magnitude_));
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, OutputScalingLooksRight) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_output_type(SpectrogramCalculatorOptions::DECIBELS);
|
||||
double output_scale = 2.5;
|
||||
options_.set_output_scale(output_scale);
|
||||
const std::vector<int> input_packet_sizes = {140};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
// Setup packets with DC input (non-zero constant value).
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_FLOAT_EQ(
|
||||
output().packets[0].Get<Matrix>()(0, 0),
|
||||
output_scale * 10.0 * std::log10(expected_dc_squared_magnitude_));
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, ComplexOutputLooksRight) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_output_type(SpectrogramCalculatorOptions::COMPLEX);
|
||||
const std::vector<int> input_packet_sizes = {140};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
// Setup packets with DC input (non-zero constant value).
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_FLOAT_EQ(std::norm(output().packets[0].Get<Eigen::MatrixXcf>()(0, 0)),
|
||||
expected_dc_squared_magnitude_);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, ComplexOutputLooksRightForImpulses) {
|
||||
const int frame_size_samples = 100;
|
||||
options_.set_frame_duration_seconds(frame_size_samples / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(0.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
options_.set_output_type(SpectrogramCalculatorOptions::COMPLEX);
|
||||
const std::vector<int> input_packet_sizes = {frame_size_samples,
|
||||
frame_size_samples};
|
||||
const std::vector<int> input_packet_impulse_offsets = {49, 50};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
|
||||
// Make two impulse packets offset one sample from each other
|
||||
SetupImpulseInputPackets(input_packet_sizes, input_packet_impulse_offsets);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
const int num_buckets =
|
||||
(audio_dsp::NextPowerOfTwo(frame_size_samples) / 2) + 1;
|
||||
const float precision = 0.01f;
|
||||
auto norm_fn = [](const std::complex<float>& cf) { return std::norm(cf); };
|
||||
|
||||
// Both impulses should have (approximately) constant power across all
|
||||
// frequency bins
|
||||
EXPECT_TRUE(output()
|
||||
.packets[0]
|
||||
.Get<Eigen::MatrixXcf>()
|
||||
.unaryExpr(norm_fn)
|
||||
.isApproxToConstant(1.0f, precision));
|
||||
EXPECT_TRUE(output()
|
||||
.packets[1]
|
||||
.Get<Eigen::MatrixXcf>()
|
||||
.unaryExpr(norm_fn)
|
||||
.isApproxToConstant(1.0f, precision));
|
||||
|
||||
// Because the second Packet's impulse is delayed by exactly one sample with
|
||||
// respect to the first Packet's impulse, the second impulse should have
|
||||
// greater phase, and in the highest frequency bin, the real part should
|
||||
// (approximately) flip sign from the first Packet to the second
|
||||
EXPECT_LT(std::arg(output().packets[0].Get<Eigen::MatrixXcf>()(1, 0)),
|
||||
std::arg(output().packets[1].Get<Eigen::MatrixXcf>()(1, 0)));
|
||||
const float highest_bucket_real_ratio =
|
||||
output().packets[0].Get<Eigen::MatrixXcf>()(num_buckets - 1, 0).real() /
|
||||
output().packets[1].Get<Eigen::MatrixXcf>()(num_buckets - 1, 0).real();
|
||||
EXPECT_NEAR(highest_bucket_real_ratio, -1.0f, precision);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, SquaredMagnitudeOutputLooksRightForNonDC) {
|
||||
const int frame_size_samples = 100;
|
||||
options_.set_frame_duration_seconds(frame_size_samples / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_output_type(SpectrogramCalculatorOptions::SQUARED_MAGNITUDE);
|
||||
const std::vector<int> input_packet_sizes = {140};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
// Make the tone have an integral number of cycles within the window
|
||||
const int target_bin = 16;
|
||||
const int fft_size = audio_dsp::NextPowerOfTwo(frame_size_samples);
|
||||
const float tone_frequency_hz = target_bin * (input_sample_rate_ / fft_size);
|
||||
SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
// For a non-DC bin, the magnitude will be split between positive and
|
||||
// negative frequency bins, so it should about be half-magnitude
|
||||
// = quarter-power.
|
||||
// It's not quite exact because of the interference from the hann(100)
|
||||
// spread from the negative-frequency half.
|
||||
EXPECT_GT(output().packets[0].Get<Matrix>()(target_bin, 0),
|
||||
0.98 * expected_dc_squared_magnitude_ / 4.0);
|
||||
EXPECT_LT(output().packets[0].Get<Matrix>()(target_bin, 0),
|
||||
1.02 * expected_dc_squared_magnitude_ / 4.0);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, ZeroOutputsForZeroInputsWithPaddingEnabled) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(true);
|
||||
const std::vector<int> input_packet_sizes = {};
|
||||
const std::vector<int> expected_output_packet_sizes = {};
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, NumChannelsIsRight) {
|
||||
const std::vector<int> input_packet_sizes = {460};
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
options_.set_allow_multichannel_input(true);
|
||||
num_input_channels_ = 3;
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
const float tone_frequency_hz = 440.0;
|
||||
SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
EXPECT_EQ(output().packets[0].Get<std::vector<Matrix>>().size(),
|
||||
num_input_channels_);
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, NumSamplesAndPacketRateAreCleared) {
|
||||
num_input_samples_ = 500;
|
||||
input_packet_rate_ = 1.0;
|
||||
const std::vector<int> input_packet_sizes = {num_input_samples_};
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(0.0);
|
||||
options_.set_pad_final_packet(false);
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
SetupConstantInputPackets(input_packet_sizes);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
const TimeSeriesHeader& output_header =
|
||||
output().header.Get<TimeSeriesHeader>();
|
||||
EXPECT_FALSE(output_header.has_num_samples());
|
||||
EXPECT_FALSE(output_header.has_packet_rate());
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, MultichannelSpectrogramSizesAreRight) {
|
||||
const std::vector<int> input_packet_sizes = {420}; // less than 10 frames
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
options_.set_allow_multichannel_input(true);
|
||||
num_input_channels_ = 10;
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
const float tone_frequency_hz = 440.0;
|
||||
SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
auto spectrograms = output().packets[0].Get<std::vector<Matrix>>();
|
||||
EXPECT_FLOAT_EQ(spectrograms.size(), num_input_channels_);
|
||||
int spectrogram_num_rows = spectrograms[0].rows();
|
||||
int spectrogram_num_cols = spectrograms[0].cols();
|
||||
for (int i = 1; i < num_input_channels_; i++) {
|
||||
EXPECT_EQ(spectrogram_num_rows, spectrograms[i].rows());
|
||||
EXPECT_EQ(spectrogram_num_cols, spectrograms[i].cols());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, MultichannelSpectrogramValuesAreRight) {
|
||||
const std::vector<int> input_packet_sizes = {
|
||||
460}; // 100 + 9*40 for 10 frames.
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_allow_multichannel_input(true);
|
||||
num_input_channels_ = 10;
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
const float tone_frequency_hz = 440.0;
|
||||
SetupMultichannelInputPackets(input_packet_sizes, tone_frequency_hz);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
auto spectrograms = output().packets[0].Get<std::vector<Matrix>>();
|
||||
int num_output_frames = spectrograms[0].cols();
|
||||
for (int i = 0; i < num_input_channels_; i++) {
|
||||
for (int frame = 0; frame < num_output_frames; ++frame) {
|
||||
if (i % 2 == 0) {
|
||||
CheckPeakFrequencyInMatrix(spectrograms[i], frame, 0);
|
||||
} else {
|
||||
CheckPeakFrequencyInMatrix(spectrograms[i], frame, tone_frequency_hz);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest, MultichannelHandlesShortInitialPacket) {
|
||||
// First packet is less than one frame, but second packet should trigger a
|
||||
// complete frame from all channels.
|
||||
const std::vector<int> input_packet_sizes = {50, 50};
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
options_.set_allow_multichannel_input(true);
|
||||
num_input_channels_ = 2;
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
const float tone_frequency_hz = 440.0;
|
||||
SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
auto spectrograms = output().packets[0].Get<std::vector<Matrix>>();
|
||||
EXPECT_FLOAT_EQ(spectrograms.size(), num_input_channels_);
|
||||
int spectrogram_num_rows = spectrograms[0].rows();
|
||||
int spectrogram_num_cols = spectrograms[0].cols();
|
||||
for (int i = 1; i < num_input_channels_; i++) {
|
||||
EXPECT_EQ(spectrogram_num_rows, spectrograms[i].rows());
|
||||
EXPECT_EQ(spectrogram_num_cols, spectrograms[i].cols());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SpectrogramCalculatorTest,
|
||||
MultichannelComplexHandlesShortInitialPacket) {
|
||||
// First packet is less than one frame, but second packet should trigger a
|
||||
// complete frame from all channels, even for complex output.
|
||||
const std::vector<int> input_packet_sizes = {50, 50};
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
options_.set_allow_multichannel_input(true);
|
||||
options_.set_output_type(SpectrogramCalculatorOptions::COMPLEX);
|
||||
num_input_channels_ = 2;
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
const float tone_frequency_hz = 440.0;
|
||||
SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
|
||||
MP_ASSERT_OK(Run());
|
||||
|
||||
CheckOutputHeadersAndTimestamps();
|
||||
auto spectrograms = output().packets[0].Get<std::vector<Eigen::MatrixXcf>>();
|
||||
EXPECT_FLOAT_EQ(spectrograms.size(), num_input_channels_);
|
||||
int spectrogram_num_rows = spectrograms[0].rows();
|
||||
int spectrogram_num_cols = spectrograms[0].cols();
|
||||
for (int i = 1; i < num_input_channels_; i++) {
|
||||
EXPECT_EQ(spectrogram_num_rows, spectrograms[i].rows());
|
||||
EXPECT_EQ(spectrogram_num_cols, spectrograms[i].cols());
|
||||
}
|
||||
}
|
||||
|
||||
void BM_ProcessDC(benchmark::State& state) {
|
||||
CalculatorGraphConfig::Node node_config;
|
||||
node_config.set_calculator("SpectrogramCalculator");
|
||||
node_config.add_input_stream("input_audio");
|
||||
node_config.add_output_stream("output_spectrogram");
|
||||
|
||||
SpectrogramCalculatorOptions* options =
|
||||
node_config.mutable_options()->MutableExtension(
|
||||
SpectrogramCalculatorOptions::ext);
|
||||
options->set_frame_duration_seconds(0.010);
|
||||
options->set_frame_overlap_seconds(0.0);
|
||||
options->set_pad_final_packet(false);
|
||||
*node_config.mutable_options()->MutableExtension(
|
||||
SpectrogramCalculatorOptions::ext) = *options;
|
||||
|
||||
int num_input_channels = 1;
|
||||
int packet_size_samples = 1600000;
|
||||
TimeSeriesHeader* header = new TimeSeriesHeader();
|
||||
header->set_sample_rate(16000.0);
|
||||
header->set_num_channels(num_input_channels);
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).header = Adopt(header);
|
||||
|
||||
Matrix* payload = new Matrix(
|
||||
Matrix::Constant(num_input_channels, packet_size_samples, 1.0));
|
||||
Timestamp timestamp = Timestamp(0);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(payload).At(timestamp));
|
||||
|
||||
for (auto _ : state) {
|
||||
ASSERT_TRUE(runner.Run().ok());
|
||||
}
|
||||
|
||||
const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0);
|
||||
const Matrix& output_matrix = output.packets[0].Get<Matrix>();
|
||||
LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
|
||||
<< output_matrix.cols();
|
||||
LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
|
||||
<< output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
|
||||
<< output_matrix(3, 0);
|
||||
}
|
||||
|
||||
BENCHMARK(BM_ProcessDC);
|
||||
|
||||
} // anonymous namespace
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,99 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Defines StabilizedLogCalculator.
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/proto_ns.h"
|
||||
#include "mediapipe/util/time_series_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "StabilizedLogCalculator"
|
||||
// input_stream: "input_time_series"
|
||||
// output_stream: "stabilized_log_time_series"
|
||||
// options {
|
||||
// [mediapipe.StabilizedLogCalculatorOptions.ext] {
|
||||
// stabilizer: .00001
|
||||
// check_nonnegativity: true
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class StabilizedLogCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<Matrix>(
|
||||
// Input stream with TimeSeriesHeader.
|
||||
);
|
||||
cc->Outputs().Index(0).Set<Matrix>(
|
||||
// Output stabilized log stream with TimeSeriesHeader.
|
||||
);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
StabilizedLogCalculatorOptions stabilized_log_calculator_options =
|
||||
cc->Options<StabilizedLogCalculatorOptions>();
|
||||
|
||||
stabilizer_ = stabilized_log_calculator_options.stabilizer();
|
||||
output_scale_ = stabilized_log_calculator_options.output_scale();
|
||||
check_nonnegativity_ =
|
||||
stabilized_log_calculator_options.check_nonnegativity();
|
||||
CHECK_GE(stabilizer_, 0.0)
|
||||
<< "stabilizer must be >= 0.0, received a value of " << stabilizer_;
|
||||
|
||||
// If the input packets have a header, propagate the header to the output.
|
||||
if (!cc->Inputs().Index(0).Header().IsEmpty()) {
|
||||
TimeSeriesHeader input_header;
|
||||
MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
|
||||
cc->Inputs().Index(0).Header(), &input_header));
|
||||
cc->Outputs().Index(0).SetHeader(
|
||||
Adopt(new TimeSeriesHeader(input_header)));
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
auto input_matrix = cc->Inputs().Index(0).Get<Matrix>();
|
||||
if (input_matrix.array().isNaN().any()) {
|
||||
return ::mediapipe::InvalidArgumentError("NaN input to log operation.");
|
||||
}
|
||||
if (check_nonnegativity_) {
|
||||
if (input_matrix.minCoeff() < 0.0) {
|
||||
return ::mediapipe::OutOfRangeError("Negative input to log operation.");
|
||||
}
|
||||
}
|
||||
std::unique_ptr<Matrix> output_frame(new Matrix(
|
||||
output_scale_ * (input_matrix.array() + stabilizer_).log().matrix()));
|
||||
cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
float stabilizer_;
|
||||
bool check_nonnegativity_;
|
||||
double output_scale_;
|
||||
};
|
||||
REGISTER_CALCULATOR(StabilizedLogCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,37 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message StabilizedLogCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional StabilizedLogCalculatorOptions ext = 101978339;
|
||||
}
|
||||
|
||||
// The calculator computes log(x + stabilizer). stabilizer must be >=
|
||||
// 0, with 0 indicating a lack of stabilization.
|
||||
optional float stabilizer = 1 [default = .00001];
|
||||
|
||||
// If true, CHECK that all input values in are >= 0. If false, the
|
||||
// code will take the log of the potentially negative input values
|
||||
// plus the stabilizer.
|
||||
optional bool check_nonnegativity = 2 [default = true];
|
||||
|
||||
// Support a fixed multiplicative scaling of the output.
|
||||
optional double output_scale = 3 [default = 1.0];
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include <cmath>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/util/time_series_test_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
const float kStabilizer = 0.1;
|
||||
const int kNumChannels = 3;
|
||||
const int kNumSamples = 10;
|
||||
|
||||
class StabilizedLogCalculatorTest
|
||||
: public TimeSeriesCalculatorTest<StabilizedLogCalculatorOptions> {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
calculator_name_ = "StabilizedLogCalculator";
|
||||
options_.set_stabilizer(kStabilizer);
|
||||
|
||||
input_sample_rate_ = 8000.0;
|
||||
num_input_channels_ = kNumChannels;
|
||||
num_input_samples_ = kNumSamples;
|
||||
}
|
||||
|
||||
void RunGraphNoReturn() { MP_ASSERT_OK(RunGraph()); }
|
||||
};
|
||||
|
||||
TEST_F(StabilizedLogCalculatorTest, BasicOperation) {
|
||||
const int kNumPackets = 5;
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
|
||||
std::vector<Matrix> input_data_matrices;
|
||||
for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
|
||||
const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
|
||||
Matrix input_data_matrix =
|
||||
Matrix::Random(kNumChannels, kNumSamples).array().abs();
|
||||
input_data_matrices.push_back(input_data_matrix);
|
||||
AppendInputPacket(new Matrix(input_data_matrix), timestamp);
|
||||
}
|
||||
|
||||
MP_ASSERT_OK(RunGraph());
|
||||
ExpectOutputHeaderEqualsInputHeader();
|
||||
for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
|
||||
ExpectApproximatelyEqual(
|
||||
(input_data_matrices[output_packet].array() + kStabilizer).log(),
|
||||
runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(StabilizedLogCalculatorTest, OutputScaleWorks) {
|
||||
const int kNumPackets = 5;
|
||||
double output_scale = 2.5;
|
||||
options_.set_output_scale(output_scale);
|
||||
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
|
||||
std::vector<Matrix> input_data_matrices;
|
||||
for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
|
||||
const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
|
||||
Matrix input_data_matrix =
|
||||
Matrix::Random(kNumChannels, kNumSamples).array().abs();
|
||||
input_data_matrices.push_back(input_data_matrix);
|
||||
AppendInputPacket(new Matrix(input_data_matrix), timestamp);
|
||||
}
|
||||
|
||||
MP_ASSERT_OK(RunGraph());
|
||||
ExpectOutputHeaderEqualsInputHeader();
|
||||
for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
|
||||
ExpectApproximatelyEqual(
|
||||
output_scale *
|
||||
((input_data_matrices[output_packet].array() + kStabilizer).log()),
|
||||
runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(StabilizedLogCalculatorTest, ZerosAreStabilized) {
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
AppendInputPacket(new Matrix(Matrix::Zero(kNumChannels, kNumSamples)),
|
||||
0 /* timestamp */);
|
||||
MP_ASSERT_OK(RunGraph());
|
||||
ExpectOutputHeaderEqualsInputHeader();
|
||||
ExpectApproximatelyEqual(
|
||||
Matrix::Constant(kNumChannels, kNumSamples, kStabilizer).array().log(),
|
||||
runner_->Outputs().Index(0).packets[0].Get<Matrix>());
|
||||
}
|
||||
|
||||
TEST_F(StabilizedLogCalculatorTest, NanValuesReturnError) {
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
AppendInputPacket(
|
||||
new Matrix(Matrix::Constant(kNumChannels, kNumSamples, std::nanf(""))),
|
||||
0 /* timestamp */);
|
||||
ASSERT_FALSE(RunGraph().ok());
|
||||
}
|
||||
|
||||
TEST_F(StabilizedLogCalculatorTest, NegativeValuesReturnError) {
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
AppendInputPacket(
|
||||
new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
|
||||
0 /* timestamp */);
|
||||
ASSERT_FALSE(RunGraph().ok());
|
||||
}
|
||||
|
||||
TEST_F(StabilizedLogCalculatorTest, NegativeValuesDoNotCheckFailIfCheckIsOff) {
|
||||
options_.set_check_nonnegativity(false);
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
AppendInputPacket(
|
||||
new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
|
||||
0 /* timestamp */);
|
||||
MP_ASSERT_OK(RunGraph());
|
||||
// Results are undefined.
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
filegroup(
|
||||
name = "test_audios",
|
||||
srcs = [
|
||||
"sine_wave_1k_44100_mono_2_sec_wav.audio",
|
||||
"sine_wave_1k_44100_stereo_2_sec_aac.audio",
|
||||
"sine_wave_1k_44100_stereo_2_sec_mp3.audio",
|
||||
"sine_wave_1k_48000_stereo_2_sec_wav.audio",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,323 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Defines TimeSeriesFramerCalculator.
|
||||
#include <math.h>
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "audio/dsp/window_functions.h"
|
||||
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/util/time_series_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// MediaPipe Calculator for framing a (vector-valued) input time series,
|
||||
// i.e. for breaking an input time series into fixed-size, possibly
|
||||
// overlapping, frames. The output stream's frame duration is
|
||||
// specified by frame_duration_seconds in the
|
||||
// TimeSeriesFramerCalculatorOptions, and the output's overlap is
|
||||
// specified by frame_overlap_seconds.
|
||||
//
|
||||
// This calculator assumes that the input timestamps refer to the
|
||||
// first sample in each Matrix. The output timestamps follow this
|
||||
// same convention.
|
||||
//
|
||||
// All output frames will have exactly the same number of samples: the number of
|
||||
// samples that approximates frame_duration_seconds most closely.
|
||||
//
|
||||
// Similarly, frame overlap is by default the (fixed) number of samples
|
||||
// approximating frame_overlap_seconds most closely. But if
|
||||
// emulate_fractional_frame_overlap is set to true, frame overlap is a variable
|
||||
// number of samples instead, such that the long-term average step between
|
||||
// frames is the difference between the (nominal) frame_duration_seconds and
|
||||
// frame_overlap_seconds.
|
||||
//
|
||||
// If pad_final_packet is true, all input samples will be emitted and the final
|
||||
// packet will be zero padded as necessary. If pad_final_packet is false, some
|
||||
// samples may be dropped at the end of the stream.
|
||||
//
|
||||
// If use_local_timestamp is true, the output packet's timestamp is based on the
|
||||
// last sample of the packet. The timestamp of this sample is inferred by
|
||||
// input_packet_timesamp + local_sample_index / sampling_rate_. If false, the
|
||||
// output packet's timestamp is based on the cumulative timestamping, which is
|
||||
// done by adopting the timestamp of the first sample of the packet and this
|
||||
// sample's timestamp is inferred by initial_input_timestamp_ +
|
||||
// cumulative_completed_samples / sample_rate_.
|
||||
class TimeSeriesFramerCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<Matrix>(
|
||||
// Input stream with TimeSeriesHeader.
|
||||
);
|
||||
cc->Outputs().Index(0).Set<Matrix>(
|
||||
// Fixed length time series Packets with TimeSeriesHeader.
|
||||
);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
// Returns FAIL if the input stream header is invalid.
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
|
||||
// Outputs as many framed packets as possible given the accumulated
|
||||
// input. Always returns OK.
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
// Flushes any remaining samples in a zero-padded packet. Always
|
||||
// returns OK.
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Adds input data to the internal buffer.
|
||||
void EnqueueInput(CalculatorContext* cc);
|
||||
// Constructs and emits framed output packets.
|
||||
void FrameOutput(CalculatorContext* cc);
|
||||
|
||||
Timestamp CurrentOutputTimestamp() {
|
||||
if (use_local_timestamp_) {
|
||||
return current_timestamp_;
|
||||
}
|
||||
return CumulativeOutputTimestamp();
|
||||
}
|
||||
|
||||
Timestamp CumulativeOutputTimestamp() {
|
||||
return initial_input_timestamp_ +
|
||||
round(cumulative_completed_samples_ / sample_rate_ *
|
||||
Timestamp::kTimestampUnitsPerSecond);
|
||||
}
|
||||
|
||||
// Returns the timestamp of a sample on a base, which is usually the time
|
||||
// stamp of a packet.
|
||||
Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
|
||||
int64 number_of_samples) {
|
||||
return timestamp_base + round(number_of_samples / sample_rate_ *
|
||||
Timestamp::kTimestampUnitsPerSecond);
|
||||
}
|
||||
|
||||
// The number of input samples to advance after the current output frame is
|
||||
// emitted.
|
||||
int next_frame_step_samples() const {
|
||||
// All numbers are in input samples.
|
||||
const int64 current_output_frame_start = static_cast<int64>(
|
||||
round(cumulative_output_frames_ * average_frame_step_samples_));
|
||||
CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
|
||||
const int64 next_output_frame_start = static_cast<int64>(
|
||||
round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
|
||||
return next_output_frame_start - current_output_frame_start;
|
||||
}
|
||||
|
||||
double sample_rate_;
|
||||
bool pad_final_packet_;
|
||||
int frame_duration_samples_;
|
||||
// The advance, in input samples, between the start of successive output
|
||||
// frames. This may be a non-integer average value if
|
||||
// emulate_fractional_frame_overlap is true.
|
||||
double average_frame_step_samples_;
|
||||
int samples_still_to_drop_;
|
||||
int64 cumulative_input_samples_;
|
||||
int64 cumulative_output_frames_;
|
||||
// "Completed" samples are samples that are no longer needed because
|
||||
// the framer has completely stepped past them (taking into account
|
||||
// any overlap).
|
||||
int64 cumulative_completed_samples_;
|
||||
Timestamp initial_input_timestamp_;
|
||||
// The current timestamp is updated along with the incoming packets.
|
||||
Timestamp current_timestamp_;
|
||||
int num_channels_;
|
||||
|
||||
// Each entry in this deque consists of a single sample, i.e. a
|
||||
// single column vector, and its timestamp.
|
||||
std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
|
||||
|
||||
bool use_window_;
|
||||
Matrix window_;
|
||||
|
||||
bool use_local_timestamp_;
|
||||
};
|
||||
REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
|
||||
|
||||
void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
|
||||
const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
|
||||
|
||||
for (int i = 0; i < input_frame.cols(); ++i) {
|
||||
sample_buffer_.emplace_back(std::make_pair(
|
||||
input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
|
||||
}
|
||||
|
||||
cumulative_input_samples_ += input_frame.cols();
|
||||
}
|
||||
|
||||
void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
|
||||
while (sample_buffer_.size() >=
|
||||
frame_duration_samples_ + samples_still_to_drop_) {
|
||||
while (samples_still_to_drop_ > 0) {
|
||||
sample_buffer_.pop_front();
|
||||
--samples_still_to_drop_;
|
||||
}
|
||||
const int frame_step_samples = next_frame_step_samples();
|
||||
std::unique_ptr<Matrix> output_frame(
|
||||
new Matrix(num_channels_, frame_duration_samples_));
|
||||
for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
|
||||
++i) {
|
||||
output_frame->col(i) = sample_buffer_.front().first;
|
||||
current_timestamp_ = sample_buffer_.front().second;
|
||||
sample_buffer_.pop_front();
|
||||
}
|
||||
const int frame_overlap_samples =
|
||||
frame_duration_samples_ - frame_step_samples;
|
||||
if (frame_overlap_samples > 0) {
|
||||
for (int i = 0; i < frame_overlap_samples; ++i) {
|
||||
output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
|
||||
current_timestamp_ = sample_buffer_[i].second;
|
||||
}
|
||||
} else {
|
||||
samples_still_to_drop_ = -frame_overlap_samples;
|
||||
}
|
||||
|
||||
if (use_window_) {
|
||||
*output_frame = (output_frame->array() * window_.array()).matrix();
|
||||
}
|
||||
|
||||
cc->Outputs().Index(0).Add(output_frame.release(),
|
||||
CurrentOutputTimestamp());
|
||||
++cumulative_output_frames_;
|
||||
cumulative_completed_samples_ += frame_step_samples;
|
||||
}
|
||||
}
|
||||
|
||||
::mediapipe::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
|
||||
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
|
||||
initial_input_timestamp_ = cc->InputTimestamp();
|
||||
current_timestamp_ = initial_input_timestamp_;
|
||||
}
|
||||
|
||||
EnqueueInput(cc);
|
||||
FrameOutput(cc);
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
|
||||
while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
|
||||
sample_buffer_.pop_front();
|
||||
--samples_still_to_drop_;
|
||||
}
|
||||
if (!sample_buffer_.empty() && pad_final_packet_) {
|
||||
std::unique_ptr<Matrix> output_frame(new Matrix);
|
||||
output_frame->setZero(num_channels_, frame_duration_samples_);
|
||||
for (int i = 0; i < sample_buffer_.size(); ++i) {
|
||||
output_frame->col(i) = sample_buffer_[i].first;
|
||||
current_timestamp_ = sample_buffer_[i].second;
|
||||
}
|
||||
|
||||
cc->Outputs().Index(0).Add(output_frame.release(),
|
||||
CurrentOutputTimestamp());
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
|
||||
TimeSeriesFramerCalculatorOptions framer_options =
|
||||
cc->Options<TimeSeriesFramerCalculatorOptions>();
|
||||
|
||||
RET_CHECK_GT(framer_options.frame_duration_seconds(), 0.0)
|
||||
<< "Invalid or missing frame_duration_seconds. "
|
||||
<< "framer_duration_seconds: \n"
|
||||
<< framer_options.frame_duration_seconds();
|
||||
RET_CHECK_LT(framer_options.frame_overlap_seconds(),
|
||||
framer_options.frame_duration_seconds())
|
||||
<< "Invalid frame_overlap_seconds. framer_overlap_seconds: \n"
|
||||
<< framer_options.frame_overlap_seconds();
|
||||
|
||||
TimeSeriesHeader input_header;
|
||||
MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
|
||||
cc->Inputs().Index(0).Header(), &input_header));
|
||||
|
||||
sample_rate_ = input_header.sample_rate();
|
||||
num_channels_ = input_header.num_channels();
|
||||
frame_duration_samples_ = time_series_util::SecondsToSamples(
|
||||
framer_options.frame_duration_seconds(), sample_rate_);
|
||||
RET_CHECK_GT(frame_duration_samples_, 0)
|
||||
<< "Frame duration of " << framer_options.frame_duration_seconds()
|
||||
<< "s too small to cover a single sample at " << sample_rate_ << " Hz ";
|
||||
if (framer_options.emulate_fractional_frame_overlap()) {
|
||||
// Frame step may be fractional.
|
||||
average_frame_step_samples_ = (framer_options.frame_duration_seconds() -
|
||||
framer_options.frame_overlap_seconds()) *
|
||||
sample_rate_;
|
||||
} else {
|
||||
// Frame step is an integer (stored in a double).
|
||||
average_frame_step_samples_ =
|
||||
frame_duration_samples_ -
|
||||
time_series_util::SecondsToSamples(
|
||||
framer_options.frame_overlap_seconds(), sample_rate_);
|
||||
}
|
||||
RET_CHECK_GE(average_frame_step_samples_, 1)
|
||||
<< "Frame step too small to cover a single sample at " << sample_rate_
|
||||
<< " Hz.";
|
||||
pad_final_packet_ = framer_options.pad_final_packet();
|
||||
|
||||
auto output_header = new TimeSeriesHeader(input_header);
|
||||
output_header->set_num_samples(frame_duration_samples_);
|
||||
if (round(average_frame_step_samples_) == average_frame_step_samples_) {
|
||||
// Only set output packet rate if it is fixed.
|
||||
output_header->set_packet_rate(sample_rate_ / average_frame_step_samples_);
|
||||
}
|
||||
cc->Outputs().Index(0).SetHeader(Adopt(output_header));
|
||||
cumulative_completed_samples_ = 0;
|
||||
cumulative_input_samples_ = 0;
|
||||
cumulative_output_frames_ = 0;
|
||||
samples_still_to_drop_ = 0;
|
||||
initial_input_timestamp_ = Timestamp::Unstarted();
|
||||
current_timestamp_ = Timestamp::Unstarted();
|
||||
|
||||
std::vector<double> window_vector;
|
||||
use_window_ = false;
|
||||
switch (framer_options.window_function()) {
|
||||
case TimeSeriesFramerCalculatorOptions::HAMMING:
|
||||
audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
|
||||
&window_vector);
|
||||
use_window_ = true;
|
||||
break;
|
||||
case TimeSeriesFramerCalculatorOptions::HANN:
|
||||
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
|
||||
&window_vector);
|
||||
use_window_ = true;
|
||||
break;
|
||||
case TimeSeriesFramerCalculatorOptions::NONE:
|
||||
break;
|
||||
}
|
||||
|
||||
if (use_window_) {
|
||||
window_ = Matrix::Ones(num_channels_, 1) *
|
||||
Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
|
||||
frame_duration_samples_)
|
||||
.cast<float>();
|
||||
}
|
||||
use_local_timestamp_ = framer_options.use_local_timestamp();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message TimeSeriesFramerCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional TimeSeriesFramerCalculatorOptions ext = 50631621;
|
||||
}
|
||||
|
||||
// Frame duration in seconds. Required. Must be greater than 0. This is
|
||||
// rounded to the nearest integer number of samples.
|
||||
optional double frame_duration_seconds = 1;
|
||||
|
||||
// Frame overlap in seconds.
|
||||
//
|
||||
// If emulate_fractional_frame_overlap is false (the default), then the frame
|
||||
// overlap is rounded to the nearest integer number of samples, and the step
|
||||
// from one frame to the next will be the difference between the number of
|
||||
// samples in a frame and the number of samples in the overlap.
|
||||
//
|
||||
// If emulate_fractional_frame_overlap is true, then frame overlap will be a
|
||||
// variable number of samples, such that the long-time average time step from
|
||||
// one frame to the next will be the difference between the (nominal, not
|
||||
// rounded) frame_duration_seconds and frame_overlap_seconds. This is useful
|
||||
// where the desired time step is not an integral number of input samples.
|
||||
//
|
||||
// A negative frame_overlap_seconds corresponds to skipping some input samples
|
||||
// between each frame of emitted samples.
|
||||
//
|
||||
// Required that frame_overlap_seconds < frame_duration_seconds.
|
||||
optional double frame_overlap_seconds = 2 [default = 0.0];
|
||||
|
||||
// See frame_overlap_seconds for semantics.
|
||||
optional bool emulate_fractional_frame_overlap = 5 [default = false];
|
||||
|
||||
// Whether to pad the final packet with zeros. If true, guarantees that all
|
||||
// input samples (other than those that fall in gaps implied by negative
|
||||
// frame_overlap_seconds) will be emitted. If set to false, any partial
|
||||
// packet at the end of the stream will be dropped.
|
||||
optional bool pad_final_packet = 3 [default = true];
|
||||
|
||||
// Optional windowing function. The default is NONE (no windowing function).
|
||||
enum WindowFunction {
|
||||
NONE = 0;
|
||||
HAMMING = 1;
|
||||
HANN = 2;
|
||||
}
|
||||
optional WindowFunction window_function = 4 [default = NONE];
|
||||
|
||||
// If use_local_timestamp is true, the output packet's timestamp is based on
|
||||
// the last sample of the packet and it's inferred from the latest input
|
||||
// packet's timestamp. If false, the output packet's timestamp is based on
|
||||
// the cumulative timestamping, which is inferred from the intial input
|
||||
// timestamp and the cumulative number of samples.
|
||||
optional bool use_local_timestamp = 6 [default = false];
|
||||
}
|
||||
@@ -0,0 +1,485 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <math.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "audio/dsp/window_functions.h"
|
||||
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/time_series_test_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
const int kInitialTimestampOffsetMicroseconds = 4;
|
||||
const int kGapBetweenPacketsInSeconds = 1;
|
||||
const int kUniversalInputPacketSize = 50;
|
||||
|
||||
class TimeSeriesFramerCalculatorTest
|
||||
: public TimeSeriesCalculatorTest<TimeSeriesFramerCalculatorOptions> {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
calculator_name_ = "TimeSeriesFramerCalculator";
|
||||
input_sample_rate_ = 4000.0;
|
||||
num_input_channels_ = 3;
|
||||
}
|
||||
|
||||
// Returns a float value with the channel and timestamp separated by
|
||||
// an order of magnitude, for easy parsing by humans.
|
||||
float TestValue(int64 timestamp_in_microseconds, int channel) {
|
||||
return timestamp_in_microseconds + channel / 10.0;
|
||||
}
|
||||
|
||||
// Caller takes ownership of the returned value.
|
||||
Matrix* NewTestFrame(int num_channels, int num_samples,
|
||||
double starting_timestamp_seconds) {
|
||||
auto matrix = new Matrix(num_channels, num_samples);
|
||||
for (int c = 0; c < num_channels; ++c) {
|
||||
for (int i = 0; i < num_samples; ++i) {
|
||||
int64 timestamp = time_series_util::SecondsToSamples(
|
||||
starting_timestamp_seconds + i / input_sample_rate_,
|
||||
Timestamp::kTimestampUnitsPerSecond);
|
||||
(*matrix)(c, i) = TestValue(timestamp, c);
|
||||
}
|
||||
}
|
||||
return matrix;
|
||||
}
|
||||
|
||||
// Initializes and runs the test graph.
|
||||
::mediapipe::Status Run() {
|
||||
InitializeGraph();
|
||||
|
||||
FillInputHeader();
|
||||
InitializeInput();
|
||||
|
||||
return RunGraph();
|
||||
}
|
||||
|
||||
// Creates test input and saves a reference copy.
|
||||
void InitializeInput() {
|
||||
concatenated_input_samples_.resize(0, num_input_channels_);
|
||||
num_input_samples_ = 0;
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
// This range of packet sizes was chosen such that some input
|
||||
// packets will be smaller than the output packet size and other
|
||||
// input packets will be larger.
|
||||
int packet_size = (i + 1) * 20;
|
||||
double timestamp_seconds = kInitialTimestampOffsetMicroseconds * 1.0e-6 +
|
||||
num_input_samples_ / input_sample_rate_;
|
||||
|
||||
Matrix* data_frame =
|
||||
NewTestFrame(num_input_channels_, packet_size, timestamp_seconds);
|
||||
|
||||
// Keep a reference copy of the input.
|
||||
//
|
||||
// conservativeResize() is needed here to preserve the existing
|
||||
// data. Eigen's resize() resizes without preserving data.
|
||||
concatenated_input_samples_.conservativeResize(
|
||||
num_input_channels_, num_input_samples_ + packet_size);
|
||||
concatenated_input_samples_.rightCols(packet_size) = *data_frame;
|
||||
num_input_samples_ += packet_size;
|
||||
|
||||
AppendInputPacket(data_frame, round(timestamp_seconds *
|
||||
Timestamp::kTimestampUnitsPerSecond));
|
||||
}
|
||||
|
||||
const int frame_duration_samples = FrameDurationSamples();
|
||||
std::vector<double> window_vector;
|
||||
switch (options_.window_function()) {
|
||||
case TimeSeriesFramerCalculatorOptions::HAMMING:
|
||||
audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples,
|
||||
&window_vector);
|
||||
break;
|
||||
case TimeSeriesFramerCalculatorOptions::HANN:
|
||||
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples,
|
||||
&window_vector);
|
||||
break;
|
||||
case TimeSeriesFramerCalculatorOptions::NONE:
|
||||
window_vector.assign(frame_duration_samples, 1.0f);
|
||||
break;
|
||||
}
|
||||
|
||||
window_ = Matrix::Ones(num_input_channels_, 1) *
|
||||
Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
|
||||
frame_duration_samples)
|
||||
.cast<float>();
|
||||
}
|
||||
|
||||
int FrameDurationSamples() {
|
||||
return time_series_util::SecondsToSamples(options_.frame_duration_seconds(),
|
||||
input_sample_rate_);
|
||||
}
|
||||
|
||||
// Checks that the values in the framed output packets matches the
|
||||
// appropriate values from the input.
|
||||
void CheckOutputPacketValues(const Matrix& actual, int packet_num,
|
||||
int frame_duration_samples,
|
||||
double frame_step_samples,
|
||||
int num_columns_to_check) {
|
||||
ASSERT_EQ(frame_duration_samples, actual.cols());
|
||||
Matrix expected = (concatenated_input_samples_
|
||||
.block(0, round(frame_step_samples * packet_num),
|
||||
num_input_channels_, num_columns_to_check)
|
||||
.array() *
|
||||
window_.leftCols(num_columns_to_check).array())
|
||||
.matrix();
|
||||
ExpectApproximatelyEqual(expected, actual.leftCols(num_columns_to_check));
|
||||
}
|
||||
|
||||
// Checks output headers, Timestamps, and values.
|
||||
void CheckOutput() {
|
||||
const int frame_duration_samples = FrameDurationSamples();
|
||||
const double frame_step_samples =
|
||||
options_.emulate_fractional_frame_overlap()
|
||||
? (options_.frame_duration_seconds() -
|
||||
options_.frame_overlap_seconds()) *
|
||||
input_sample_rate_
|
||||
: frame_duration_samples -
|
||||
time_series_util::SecondsToSamples(
|
||||
options_.frame_overlap_seconds(), input_sample_rate_);
|
||||
|
||||
TimeSeriesHeader expected_header = input().header.Get<TimeSeriesHeader>();
|
||||
expected_header.set_num_samples(frame_duration_samples);
|
||||
if (!options_.emulate_fractional_frame_overlap() ||
|
||||
frame_step_samples == round(frame_step_samples)) {
|
||||
expected_header.set_packet_rate(input_sample_rate_ / frame_step_samples);
|
||||
}
|
||||
ExpectOutputHeaderEquals(expected_header);
|
||||
|
||||
int num_full_packets = output().packets.size();
|
||||
if (options_.pad_final_packet()) {
|
||||
num_full_packets -= 1;
|
||||
}
|
||||
|
||||
for (int packet_num = 0; packet_num < num_full_packets; ++packet_num) {
|
||||
const Packet& packet = output().packets[packet_num];
|
||||
CheckOutputPacketValues(packet.Get<Matrix>(), packet_num,
|
||||
frame_duration_samples, frame_step_samples,
|
||||
frame_duration_samples);
|
||||
}
|
||||
|
||||
// What is the effective time index of the final sample emitted?
|
||||
// This includes accounting for the gaps when overlap is negative.
|
||||
const int num_unique_output_samples =
|
||||
round((output().packets.size() - 1) * frame_step_samples) +
|
||||
frame_duration_samples;
|
||||
LOG(INFO) << "packets.size()=" << output().packets.size()
|
||||
<< " frame_duration_samples=" << frame_duration_samples
|
||||
<< " frame_step_samples=" << frame_step_samples
|
||||
<< " num_input_samples_=" << num_input_samples_
|
||||
<< " num_unique_output_samples=" << num_unique_output_samples;
|
||||
const int num_padding_samples =
|
||||
num_unique_output_samples - num_input_samples_;
|
||||
if (options_.pad_final_packet()) {
|
||||
EXPECT_LT(num_padding_samples, frame_duration_samples);
|
||||
// If the input ended during the dropped samples between the end of
|
||||
// the last emitted frame and where the next one would begin, there
|
||||
// can be fewer unique output points than input points, even with
|
||||
// padding.
|
||||
const int max_dropped_samples =
|
||||
static_cast<int>(ceil(frame_step_samples - frame_duration_samples));
|
||||
EXPECT_GE(num_padding_samples, std::min(0, -max_dropped_samples));
|
||||
|
||||
if (num_padding_samples > 0) {
|
||||
// Check the non-padded part of the final packet.
|
||||
const Matrix& final_matrix = output().packets.back().Get<Matrix>();
|
||||
CheckOutputPacketValues(final_matrix, num_full_packets,
|
||||
frame_duration_samples, frame_step_samples,
|
||||
frame_duration_samples - num_padding_samples);
|
||||
// Check the padded part of the final packet.
|
||||
EXPECT_EQ(
|
||||
Matrix::Zero(num_input_channels_, num_padding_samples),
|
||||
final_matrix.block(0, frame_duration_samples - num_padding_samples,
|
||||
num_input_channels_, num_padding_samples));
|
||||
}
|
||||
} else {
|
||||
EXPECT_GT(num_padding_samples, -frame_duration_samples);
|
||||
EXPECT_LE(num_padding_samples, 0);
|
||||
}
|
||||
}
|
||||
|
||||
int num_input_samples_;
|
||||
Matrix concatenated_input_samples_;
|
||||
Matrix window_;
|
||||
};
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, IntegerSampleDurationNoOverlap) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
MP_ASSERT_OK(Run());
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest,
|
||||
IntegerSampleDurationNoOverlapHammingWindow) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_window_function(TimeSeriesFramerCalculatorOptions::HAMMING);
|
||||
MP_ASSERT_OK(Run());
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest,
|
||||
IntegerSampleDurationNoOverlapHannWindow) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_window_function(TimeSeriesFramerCalculatorOptions::HANN);
|
||||
MP_ASSERT_OK(Run());
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, IntegerSampleDurationAndOverlap) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(40.0 / input_sample_rate_);
|
||||
MP_ASSERT_OK(Run());
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, NonintegerSampleDurationAndOverlap) {
|
||||
options_.set_frame_duration_seconds(98.5 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(38.4 / input_sample_rate_);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapExactFrames) {
|
||||
// Negative overlap means to drop samples between frames.
|
||||
// 100 samples per frame plus a skip of 10 samples will be 10 full frames in
|
||||
// the 1100 input samples.
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(-10.0 / input_sample_rate_);
|
||||
MP_ASSERT_OK(Run());
|
||||
EXPECT_EQ(output().packets.size(), 10);
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapExactFramesLessSkip) {
|
||||
// 100 samples per frame plus a skip of 100 samples will be 6 full frames in
|
||||
// the 1100 input samples.
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(-100.0 / input_sample_rate_);
|
||||
MP_ASSERT_OK(Run());
|
||||
EXPECT_EQ(output().packets.size(), 6);
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapWithPadding) {
|
||||
// 150 samples per frame plus a skip of 50 samples will require some padding
|
||||
// on the sixth and last frame given 1100 sample input.
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(-100.0 / input_sample_rate_);
|
||||
MP_ASSERT_OK(Run());
|
||||
EXPECT_EQ(output().packets.size(), 6);
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, FixedFrameOverlap) {
|
||||
// Frame of 30 samples with step of 11.4 samples (rounded to 11 samples)
|
||||
// results in ceil((1100 - 30) / 11) + 1 = 99 packets.
|
||||
options_.set_frame_duration_seconds(30 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds((30.0 - 11.4) / input_sample_rate_);
|
||||
MP_ASSERT_OK(Run());
|
||||
EXPECT_EQ(output().packets.size(), 99);
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, VariableFrameOverlap) {
|
||||
// Frame of 30 samples with step of 11.4 samples (not rounded)
|
||||
// results in ceil((1100 - 30) / 11.4) + 1 = 95 packets.
|
||||
options_.set_frame_duration_seconds(30 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds((30 - 11.4) / input_sample_rate_);
|
||||
options_.set_emulate_fractional_frame_overlap(true);
|
||||
MP_ASSERT_OK(Run());
|
||||
EXPECT_EQ(output().packets.size(), 95);
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, VariableFrameSkip) {
|
||||
// Frame of 30 samples with step of 41.4 samples (not rounded)
|
||||
// results in ceil((1100 - 30) / 41.4) + 1 = 27 packets.
|
||||
options_.set_frame_duration_seconds(30 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds((30 - 41.4) / input_sample_rate_);
|
||||
options_.set_emulate_fractional_frame_overlap(true);
|
||||
MP_ASSERT_OK(Run());
|
||||
EXPECT_EQ(output().packets.size(), 27);
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest, NoFinalPacketPadding) {
|
||||
options_.set_frame_duration_seconds(98.5 / input_sample_rate_);
|
||||
options_.set_pad_final_packet(false);
|
||||
|
||||
MP_ASSERT_OK(Run());
|
||||
CheckOutput();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest,
|
||||
FrameRateHigherThanSampleRate_FrameDurationTooLow) {
|
||||
// Try to produce a frame rate 10 times the input sample rate by using a
|
||||
// a frame duration that is too small and covers only 0.1 samples.
|
||||
options_.set_frame_duration_seconds(1 / (10 * input_sample_rate_));
|
||||
options_.set_frame_overlap_seconds(0.0);
|
||||
EXPECT_FALSE(Run().ok());
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTest,
|
||||
FrameRateHigherThanSampleRate_FrameStepTooLow) {
|
||||
// Try to produce a frame rate 10 times the input sample rate by using
|
||||
// a frame overlap that is too high and produces frame steps (difference
|
||||
// between duration and overlap) of 0.1 samples.
|
||||
options_.set_frame_duration_seconds(10.0 / input_sample_rate_);
|
||||
options_.set_frame_overlap_seconds(9.9 / input_sample_rate_);
|
||||
EXPECT_FALSE(Run().ok());
|
||||
}
|
||||
|
||||
// A simple test class to do windowing sanity checks. Tests from this
|
||||
// class input a single packet of all ones, and check the average
|
||||
// value of the single output packet. This is useful as a sanity check
|
||||
// that the correct windows are applied.
|
||||
class TimeSeriesFramerCalculatorWindowingSanityTest
|
||||
: public TimeSeriesFramerCalculatorTest {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
TimeSeriesFramerCalculatorTest::SetUp();
|
||||
num_input_channels_ = 1;
|
||||
}
|
||||
|
||||
void RunAndTestSinglePacketAverage(float expected_average) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
InitializeGraph();
|
||||
FillInputHeader();
|
||||
AppendInputPacket(new Matrix(Matrix::Ones(1, FrameDurationSamples())),
|
||||
kInitialTimestampOffsetMicroseconds);
|
||||
MP_ASSERT_OK(RunGraph());
|
||||
ASSERT_EQ(1, output().packets.size());
|
||||
ASSERT_NEAR(expected_average * FrameDurationSamples(),
|
||||
output().packets[0].Get<Matrix>().sum(), 1e-5);
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest, NoWindowSanityCheck) {
|
||||
RunAndTestSinglePacketAverage(1.0f);
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest,
|
||||
HammingWindowSanityCheck) {
|
||||
options_.set_window_function(TimeSeriesFramerCalculatorOptions::HAMMING);
|
||||
RunAndTestSinglePacketAverage(0.54f);
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest, HannWindowSanityCheck) {
|
||||
options_.set_window_function(TimeSeriesFramerCalculatorOptions::HANN);
|
||||
RunAndTestSinglePacketAverage(0.5f);
|
||||
}
|
||||
|
||||
// A simple test class that checks the local packet time stamp. This class
|
||||
// generate a series of packets with and without gaps between packets and tests
|
||||
// the behavior with cumulative timestamping and local packet timestamping.
|
||||
class TimeSeriesFramerCalculatorTimestampingTest
|
||||
: public TimeSeriesFramerCalculatorTest {
|
||||
protected:
|
||||
// Creates test input and saves a reference copy.
|
||||
void InitializeInputForTimeStampingTest() {
|
||||
concatenated_input_samples_.resize(0, num_input_channels_);
|
||||
num_input_samples_ = 0;
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
// This range of packet sizes was chosen such that some input
|
||||
// packets will be smaller than the output packet size and other
|
||||
// input packets will be larger.
|
||||
int packet_size = kUniversalInputPacketSize;
|
||||
double timestamp_seconds = kInitialTimestampOffsetMicroseconds * 1.0e-6 +
|
||||
num_input_samples_ / input_sample_rate_;
|
||||
if (options_.use_local_timestamp()) {
|
||||
timestamp_seconds += kGapBetweenPacketsInSeconds * i;
|
||||
}
|
||||
|
||||
Matrix* data_frame =
|
||||
NewTestFrame(num_input_channels_, packet_size, timestamp_seconds);
|
||||
|
||||
AppendInputPacket(data_frame, round(timestamp_seconds *
|
||||
Timestamp::kTimestampUnitsPerSecond));
|
||||
num_input_samples_ += packet_size;
|
||||
}
|
||||
}
|
||||
|
||||
void CheckOutputTimestamps() {
|
||||
int num_full_packets = output().packets.size();
|
||||
if (options_.pad_final_packet()) {
|
||||
num_full_packets -= 1;
|
||||
}
|
||||
|
||||
int64 num_samples = 0;
|
||||
for (int packet_num = 0; packet_num < num_full_packets; ++packet_num) {
|
||||
const Packet& packet = output().packets[packet_num];
|
||||
num_samples += FrameDurationSamples();
|
||||
double expected_timestamp =
|
||||
options_.use_local_timestamp()
|
||||
? GetExpectedLocalTimestampForSample(num_samples - 1)
|
||||
: GetExpectedCumulativeTimestamp(num_samples - 1);
|
||||
ASSERT_NEAR(packet.Timestamp().Seconds(), expected_timestamp, 1e-10);
|
||||
}
|
||||
}
|
||||
|
||||
::mediapipe::Status RunTimestampTest() {
|
||||
InitializeGraph();
|
||||
InitializeInputForTimeStampingTest();
|
||||
FillInputHeader();
|
||||
return RunGraph();
|
||||
}
|
||||
|
||||
private:
|
||||
// Returns the timestamp in seconds based on local timestamping.
|
||||
double GetExpectedLocalTimestampForSample(int sample_index) {
|
||||
return kInitialTimestampOffsetMicroseconds * 1.0e-6 +
|
||||
sample_index / input_sample_rate_ +
|
||||
(sample_index / kUniversalInputPacketSize) *
|
||||
kGapBetweenPacketsInSeconds;
|
||||
}
|
||||
|
||||
// Returns the timestamp inseconds based on cumulative timestamping.
|
||||
double GetExpectedCumulativeTimestamp(int sample_index) {
|
||||
return kInitialTimestampOffsetMicroseconds * 1.0e-6 +
|
||||
sample_index / FrameDurationSamples() * FrameDurationSamples() /
|
||||
input_sample_rate_;
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseLocalTimeStamp) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_use_local_timestamp(true);
|
||||
|
||||
MP_ASSERT_OK(RunTimestampTest());
|
||||
CheckOutputTimestamps();
|
||||
}
|
||||
|
||||
TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseCumulativeTimeStamp) {
|
||||
options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
|
||||
options_.set_use_local_timestamp(false);
|
||||
|
||||
MP_ASSERT_OK(RunTimestampTest());
|
||||
CheckOutputTimestamps();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -13,11 +13,32 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
proto_library(
|
||||
name = "concatenate_vector_calculator_proto",
|
||||
srcs = ["concatenate_vector_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "dequantize_byte_array_calculator_proto",
|
||||
srcs = ["dequantize_byte_array_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "packet_cloner_calculator_proto",
|
||||
srcs = ["packet_cloner_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "packet_resampler_calculator_proto",
|
||||
@@ -26,14 +47,329 @@ proto_library(
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "packet_thinner_calculator_proto",
|
||||
srcs = ["packet_thinner_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "split_vector_calculator_proto",
|
||||
srcs = ["split_vector_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "quantize_float_vector_calculator_proto",
|
||||
srcs = ["quantize_float_vector_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "sequence_shift_calculator_proto",
|
||||
srcs = ["sequence_shift_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "gate_calculator_proto",
|
||||
srcs = ["gate_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "constant_side_packet_calculator_proto",
|
||||
srcs = ["constant_side_packet_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "clip_vector_size_calculator_proto",
|
||||
srcs = ["clip_vector_size_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "packet_cloner_calculator_cc_proto",
|
||||
srcs = ["packet_cloner_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":packet_cloner_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "packet_resampler_calculator_cc_proto",
|
||||
srcs = ["packet_resampler_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//mediapipe:__subpackages__"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":packet_resampler_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "packet_thinner_calculator_cc_proto",
|
||||
srcs = ["packet_thinner_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":packet_thinner_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "split_vector_calculator_cc_proto",
|
||||
srcs = ["split_vector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":split_vector_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "concatenate_vector_calculator_cc_proto",
|
||||
srcs = ["concatenate_vector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":concatenate_vector_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "clip_vector_size_calculator_cc_proto",
|
||||
srcs = ["clip_vector_size_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":clip_vector_size_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "dequantize_byte_array_calculator_cc_proto",
|
||||
srcs = ["dequantize_byte_array_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":dequantize_byte_array_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "quantize_float_vector_calculator_cc_proto",
|
||||
srcs = ["quantize_float_vector_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":quantize_float_vector_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "sequence_shift_calculator_cc_proto",
|
||||
srcs = ["sequence_shift_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":sequence_shift_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "gate_calculator_cc_proto",
|
||||
srcs = ["gate_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":gate_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "constant_side_packet_calculator_cc_proto",
|
||||
srcs = ["constant_side_packet_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":constant_side_packet_calculator_proto"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "add_header_calculator",
|
||||
srcs = ["add_header_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "add_header_calculator_test",
|
||||
size = "small",
|
||||
srcs = ["add_header_calculator_test.cc"],
|
||||
deps = [
|
||||
":add_header_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "begin_loop_calculator",
|
||||
srcs = ["begin_loop_calculator.cc"],
|
||||
hdrs = ["begin_loop_calculator.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/memory",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "end_loop_calculator",
|
||||
srcs = ["end_loop_calculator.cc"],
|
||||
hdrs = ["end_loop_calculator.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:render_data_cc_proto",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "begin_end_loop_calculator_graph_test",
|
||||
srcs = ["begin_end_loop_calculator_graph_test.cc"],
|
||||
deps = [
|
||||
":begin_loop_calculator",
|
||||
":end_loop_calculator",
|
||||
"//mediapipe/calculators/core:packet_cloner_calculator",
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/memory",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "concatenate_vector_calculator",
|
||||
srcs = ["concatenate_vector_calculator.cc"],
|
||||
hdrs = ["concatenate_vector_calculator.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":concatenate_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//mediapipe:ios": [],
|
||||
"//conditions:default": [
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "concatenate_detection_vector_calculator",
|
||||
srcs = ["concatenate_detection_vector_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":concatenate_vector_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "concatenate_vector_calculator_test",
|
||||
srcs = ["concatenate_vector_calculator_test.cc"],
|
||||
deps = [
|
||||
":concatenate_vector_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "clip_vector_size_calculator",
|
||||
srcs = ["clip_vector_size_calculator.cc"],
|
||||
hdrs = ["clip_vector_size_calculator.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":clip_vector_size_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "clip_detection_vector_size_calculator",
|
||||
srcs = ["clip_detection_vector_size_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":clip_vector_size_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "clip_vector_size_calculator_test",
|
||||
srcs = ["clip_vector_size_calculator_test.cc"],
|
||||
deps = [
|
||||
":clip_vector_size_calculator",
|
||||
"//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "counting_source_calculator",
|
||||
srcs = ["counting_source_calculator.cc"],
|
||||
@@ -62,6 +398,38 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "matrix_multiply_calculator",
|
||||
srcs = ["matrix_multiply_calculator.cc"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "matrix_subtract_calculator",
|
||||
srcs = ["matrix_subtract_calculator.cc"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "mux_calculator",
|
||||
srcs = ["mux_calculator.cc"],
|
||||
@@ -83,12 +451,68 @@ cc_library(
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [
|
||||
":packet_cloner_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "packet_inner_join_calculator",
|
||||
srcs = ["packet_inner_join_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "packet_inner_join_calculator_test",
|
||||
srcs = ["packet_inner_join_calculator_test.cc"],
|
||||
deps = [
|
||||
":packet_inner_join_calculator",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "packet_thinner_calculator",
|
||||
srcs = ["packet_thinner_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:video_stream_header",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "packet_thinner_calculator_test",
|
||||
srcs = ["packet_thinner_calculator_test.cc"],
|
||||
deps = [
|
||||
":packet_thinner_calculator",
|
||||
"//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:video_stream_header",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "pass_through_calculator",
|
||||
srcs = ["pass_through_calculator.cc"],
|
||||
@@ -145,8 +569,8 @@ cc_library(
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "real_time_flow_limiter_calculator",
|
||||
srcs = ["real_time_flow_limiter_calculator.cc"],
|
||||
name = "flow_limiter_calculator",
|
||||
srcs = ["flow_limiter_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
@@ -160,6 +584,32 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "string_to_int_calculator",
|
||||
srcs = ["string_to_int_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "side_packet_to_stream_calculator",
|
||||
srcs = ["side_packet_to_stream_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "immediate_mux_calculator_test",
|
||||
srcs = ["immediate_mux_calculator_test.cc"],
|
||||
@@ -184,6 +634,7 @@ cc_test(
|
||||
cc_library(
|
||||
name = "packet_resampler_calculator",
|
||||
srcs = ["packet_resampler_calculator.cc"],
|
||||
hdrs = ["packet_resampler_calculator.h"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
@@ -191,36 +642,33 @@ cc_library(
|
||||
"//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework/deps:mathutil",
|
||||
"//mediapipe/framework/deps:random",
|
||||
"//mediapipe/framework/formats:video_stream_header",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:options_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"//mediapipe/framework/deps:mathutil",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
] + select({
|
||||
"//conditions:default": [
|
||||
"//mediapipe/framework/deps:random",
|
||||
],
|
||||
}),
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "packet_resampler_calculator_test",
|
||||
timeout = "short",
|
||||
srcs = ["packet_resampler_calculator_test.cc"],
|
||||
srcs = [
|
||||
"packet_resampler_calculator_test.cc",
|
||||
],
|
||||
deps = [
|
||||
":packet_resampler_calculator",
|
||||
"//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:video_stream_header",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
@@ -245,10 +693,43 @@ cc_test(
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "real_time_flow_limiter_calculator_test",
|
||||
srcs = ["real_time_flow_limiter_calculator_test.cc"],
|
||||
name = "matrix_multiply_calculator_test",
|
||||
srcs = ["matrix_multiply_calculator_test.cc"],
|
||||
visibility = ["//visibility:private"],
|
||||
deps = [
|
||||
":real_time_flow_limiter_calculator",
|
||||
":matrix_multiply_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "matrix_subtract_calculator_test",
|
||||
srcs = ["matrix_subtract_calculator_test.cc"],
|
||||
visibility = ["//visibility:private"],
|
||||
deps = [
|
||||
":matrix_subtract_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "flow_limiter_calculator_test",
|
||||
srcs = ["flow_limiter_calculator_test.cc"],
|
||||
deps = [
|
||||
":flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:counting_source_calculator",
|
||||
"//mediapipe/calculators/core:pass_through_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
@@ -264,3 +745,262 @@ cc_test(
|
||||
"@com_google_absl//absl/time",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "split_vector_calculator",
|
||||
srcs = ["split_vector_calculator.cc"],
|
||||
hdrs = ["split_vector_calculator.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":split_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:resource_util",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//mediapipe:ios": [],
|
||||
"//conditions:default": [
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "split_vector_calculator_test",
|
||||
srcs = ["split_vector_calculator_test.cc"],
|
||||
deps = [
|
||||
":split_vector_calculator",
|
||||
":split_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "dequantize_byte_array_calculator",
|
||||
srcs = ["dequantize_byte_array_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":dequantize_byte_array_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "dequantize_byte_array_calculator_test",
|
||||
srcs = ["dequantize_byte_array_calculator_test.cc"],
|
||||
deps = [
|
||||
":dequantize_byte_array_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "quantize_float_vector_calculator",
|
||||
srcs = ["quantize_float_vector_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":quantize_float_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "quantize_float_vector_calculator_test",
|
||||
srcs = ["quantize_float_vector_calculator_test.cc"],
|
||||
deps = [
|
||||
":quantize_float_vector_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "sequence_shift_calculator",
|
||||
srcs = ["sequence_shift_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":sequence_shift_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "sequence_shift_calculator_test",
|
||||
srcs = ["sequence_shift_calculator_test.cc"],
|
||||
deps = [
|
||||
":sequence_shift_calculator",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "gate_calculator",
|
||||
srcs = ["gate_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":gate_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:header_util",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "gate_calculator_test",
|
||||
srcs = ["gate_calculator_test.cc"],
|
||||
deps = [
|
||||
":gate_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "matrix_to_vector_calculator",
|
||||
srcs = ["matrix_to_vector_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:status_util",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "matrix_to_vector_calculator_test",
|
||||
srcs = ["matrix_to_vector_calculator_test.cc"],
|
||||
deps = [
|
||||
":matrix_to_vector_calculator",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"//mediapipe/util:time_series_util",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "merge_calculator",
|
||||
srcs = ["merge_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "merge_calculator_test",
|
||||
srcs = ["merge_calculator_test.cc"],
|
||||
deps = [
|
||||
":merge_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "stream_to_side_packet_calculator",
|
||||
srcs = ["stream_to_side_packet_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "stream_to_side_packet_calculator_test",
|
||||
srcs = ["stream_to_side_packet_calculator_test.cc"],
|
||||
deps = [
|
||||
":stream_to_side_packet_calculator",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/memory",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "constant_side_packet_calculator",
|
||||
srcs = ["constant_side_packet_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":constant_side_packet_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "constant_side_packet_calculator_test",
|
||||
srcs = ["constant_side_packet_calculator_test.cc"],
|
||||
deps = [
|
||||
":constant_side_packet_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Attach the header from a stream or side input to another stream.
|
||||
//
|
||||
// The header stream (tag HEADER) must not have any packets in it.
|
||||
//
|
||||
// Before using this calculator, please think about changing your
|
||||
// calculator to not need a header or to accept a separate stream with
|
||||
// a header, that would be more future proof.
|
||||
//
|
||||
// Example usage 1:
|
||||
// node {
|
||||
// calculator: "AddHeaderCalculator"
|
||||
// input_stream: "DATA:audio"
|
||||
// input_stream: "HEADER:audio_header"
|
||||
// output_stream: "audio_with_header"
|
||||
// }
|
||||
//
|
||||
// Example usage 2:
|
||||
// node {
|
||||
// calculator: "AddHeaderCalculator"
|
||||
// input_stream: "DATA:audio"
|
||||
// input_side_packet: "HEADER:audio_header"
|
||||
// output_stream: "audio_with_header"
|
||||
// }
|
||||
//
|
||||
class AddHeaderCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
bool has_side_input = false;
|
||||
bool has_header_stream = false;
|
||||
if (cc->InputSidePackets().HasTag("HEADER")) {
|
||||
cc->InputSidePackets().Tag("HEADER").SetAny();
|
||||
has_side_input = true;
|
||||
}
|
||||
if (cc->Inputs().HasTag("HEADER")) {
|
||||
cc->Inputs().Tag("HEADER").SetNone();
|
||||
has_header_stream = true;
|
||||
}
|
||||
if (has_side_input == has_header_stream) {
|
||||
return mediapipe::InvalidArgumentError(
|
||||
"Header must be provided via exactly one of side input and input "
|
||||
"stream");
|
||||
}
|
||||
cc->Inputs().Tag("DATA").SetAny();
|
||||
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Tag("DATA"));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
Packet header;
|
||||
if (cc->InputSidePackets().HasTag("HEADER")) {
|
||||
header = cc->InputSidePackets().Tag("HEADER");
|
||||
}
|
||||
if (cc->Inputs().HasTag("HEADER")) {
|
||||
header = cc->Inputs().Tag("HEADER").Header();
|
||||
}
|
||||
if (!header.IsEmpty()) {
|
||||
cc->Outputs().Index(0).SetHeader(header);
|
||||
}
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
cc->Outputs().Index(0).AddPacket(cc->Inputs().Tag("DATA").Value());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(AddHeaderCalculator);
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,159 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class AddHeaderCalculatorTest : public ::testing::Test {};
|
||||
|
||||
TEST_F(AddHeaderCalculatorTest, HeaderStream) {
|
||||
CalculatorGraphConfig::Node node;
|
||||
node.set_calculator("AddHeaderCalculator");
|
||||
node.add_input_stream("HEADER:header_stream");
|
||||
node.add_input_stream("DATA:data_stream");
|
||||
node.add_output_stream("merged_stream");
|
||||
|
||||
CalculatorRunner runner(node);
|
||||
|
||||
// Set header and add 5 packets.
|
||||
runner.MutableInputs()->Tag("HEADER").header =
|
||||
Adopt(new std::string("my_header"));
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
|
||||
}
|
||||
|
||||
// Run calculator.
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
ASSERT_EQ(1, runner.Outputs().NumEntries());
|
||||
|
||||
// Test output.
|
||||
EXPECT_EQ(std::string("my_header"),
|
||||
runner.Outputs().Index(0).header.Get<std::string>());
|
||||
const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
|
||||
ASSERT_EQ(5, output_packets.size());
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
const int val = output_packets[i].Get<int>();
|
||||
EXPECT_EQ(i, val);
|
||||
EXPECT_EQ(Timestamp(i * 1000), output_packets[i].Timestamp());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(AddHeaderCalculatorTest, HandlesEmptyHeaderStream) {
|
||||
CalculatorGraphConfig::Node node;
|
||||
node.set_calculator("AddHeaderCalculator");
|
||||
node.add_input_stream("HEADER:header_stream");
|
||||
node.add_input_stream("DATA:data_stream");
|
||||
node.add_output_stream("merged_stream");
|
||||
|
||||
CalculatorRunner runner(node);
|
||||
|
||||
// No header and no packets.
|
||||
// Run calculator.
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
EXPECT_TRUE(runner.Outputs().Index(0).header.IsEmpty());
|
||||
}
|
||||
|
||||
TEST_F(AddHeaderCalculatorTest, NoPacketsOnHeaderStream) {
|
||||
CalculatorGraphConfig::Node node;
|
||||
node.set_calculator("AddHeaderCalculator");
|
||||
node.add_input_stream("HEADER:header_stream");
|
||||
node.add_input_stream("DATA:data_stream");
|
||||
node.add_output_stream("merged_stream");
|
||||
|
||||
CalculatorRunner runner(node);
|
||||
|
||||
// Set header and add 5 packets.
|
||||
runner.MutableInputs()->Tag("HEADER").header =
|
||||
Adopt(new std::string("my_header"));
|
||||
runner.MutableInputs()->Tag("HEADER").packets.push_back(
|
||||
Adopt(new std::string("not allowed")));
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
|
||||
}
|
||||
|
||||
// Run calculator.
|
||||
ASSERT_FALSE(runner.Run().ok());
|
||||
}
|
||||
|
||||
TEST_F(AddHeaderCalculatorTest, InputSidePacket) {
|
||||
CalculatorGraphConfig::Node node;
|
||||
node.set_calculator("AddHeaderCalculator");
|
||||
node.add_input_stream("DATA:data_stream");
|
||||
node.add_output_stream("merged_stream");
|
||||
node.add_input_side_packet("HEADER:header");
|
||||
|
||||
CalculatorRunner runner(node);
|
||||
|
||||
// Set header and add 5 packets.
|
||||
runner.MutableSidePackets()->Tag("HEADER") =
|
||||
Adopt(new std::string("my_header"));
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
|
||||
}
|
||||
|
||||
// Run calculator.
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
ASSERT_EQ(1, runner.Outputs().NumEntries());
|
||||
|
||||
// Test output.
|
||||
EXPECT_EQ(std::string("my_header"),
|
||||
runner.Outputs().Index(0).header.Get<std::string>());
|
||||
const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
|
||||
ASSERT_EQ(5, output_packets.size());
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
const int val = output_packets[i].Get<int>();
|
||||
EXPECT_EQ(i, val);
|
||||
EXPECT_EQ(Timestamp(i * 1000), output_packets[i].Timestamp());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(AddHeaderCalculatorTest, UsingBothSideInputAndStream) {
|
||||
CalculatorGraphConfig::Node node;
|
||||
node.set_calculator("AddHeaderCalculator");
|
||||
node.add_input_stream("HEADER:header_stream");
|
||||
node.add_input_stream("DATA:data_stream");
|
||||
node.add_output_stream("merged_stream");
|
||||
node.add_input_side_packet("HEADER:header");
|
||||
|
||||
CalculatorRunner runner(node);
|
||||
|
||||
// Set both headers and add 5 packets.
|
||||
runner.MutableSidePackets()->Tag("HEADER") =
|
||||
Adopt(new std::string("my_header"));
|
||||
runner.MutableSidePackets()->Tag("HEADER") =
|
||||
Adopt(new std::string("my_header"));
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
|
||||
}
|
||||
|
||||
// Run should fail because header can only be provided one way.
|
||||
EXPECT_EQ(runner.Run().code(), ::mediapipe::InvalidArgumentError("").code());
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,335 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/calculators/core/begin_loop_calculator.h"
|
||||
#include "mediapipe/calculators/core/end_loop_calculator.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
typedef BeginLoopCalculator<std::vector<int>> BeginLoopIntegerCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopIntegerCalculator);
|
||||
|
||||
class IncrementCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<int>();
|
||||
cc->Outputs().Index(0).Set<int>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
const int& input_int = cc->Inputs().Index(0).Get<int>();
|
||||
auto output_int = absl::make_unique<int>(input_int + 1);
|
||||
cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(IncrementCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<int>> EndLoopIntegersCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopIntegersCalculator);
|
||||
|
||||
class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
|
||||
protected:
|
||||
BeginEndLoopCalculatorGraphTest() {
|
||||
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
num_threads: 4
|
||||
input_stream: "ints"
|
||||
node {
|
||||
calculator: "BeginLoopIntegerCalculator"
|
||||
input_stream: "ITERABLE:ints"
|
||||
output_stream: "ITEM:int"
|
||||
output_stream: "BATCH_END:timestamp"
|
||||
}
|
||||
node {
|
||||
calculator: "IncrementCalculator"
|
||||
input_stream: "int"
|
||||
output_stream: "int_plus_one"
|
||||
}
|
||||
node {
|
||||
calculator: "EndLoopIntegersCalculator"
|
||||
input_stream: "ITEM:int_plus_one"
|
||||
input_stream: "BATCH_END:timestamp"
|
||||
output_stream: "ITERABLE:ints_plus_one"
|
||||
}
|
||||
)");
|
||||
tool::AddVectorSink("ints_plus_one", &graph_config_, &output_packets_);
|
||||
}
|
||||
|
||||
CalculatorGraphConfig graph_config_;
|
||||
std::vector<Packet> output_packets_;
|
||||
};
|
||||
|
||||
TEST_F(BeginEndLoopCalculatorGraphTest, SingleEmptyVector) {
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config_));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
auto input_vector = absl::make_unique<std::vector<int>>();
|
||||
Timestamp input_timestamp = Timestamp(0);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector.release()).At(input_timestamp)));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
// EndLoopCalc will forward the timestamp bound because there are no elements
|
||||
// in collection to output.
|
||||
ASSERT_EQ(0, output_packets_.size());
|
||||
|
||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(BeginEndLoopCalculatorGraphTest, SingleNonEmptyVector) {
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config_));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
auto input_vector = absl::make_unique<std::vector<int>>();
|
||||
input_vector->emplace_back(0);
|
||||
input_vector->emplace_back(1);
|
||||
input_vector->emplace_back(2);
|
||||
Timestamp input_timestamp = Timestamp(0);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector.release()).At(input_timestamp)));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
ASSERT_EQ(1, output_packets_.size());
|
||||
EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp());
|
||||
std::vector<int> expected_output_vector = {1, 2, 3};
|
||||
EXPECT_EQ(expected_output_vector, output_packets_[0].Get<std::vector<int>>());
|
||||
|
||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) {
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config_));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
|
||||
auto input_vector0 = absl::make_unique<std::vector<int>>();
|
||||
input_vector0->emplace_back(0);
|
||||
input_vector0->emplace_back(1);
|
||||
Timestamp input_timestamp0 = Timestamp(0);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector0.release()).At(input_timestamp0)));
|
||||
|
||||
auto input_vector1 = absl::make_unique<std::vector<int>>();
|
||||
Timestamp input_timestamp1 = Timestamp(1);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector1.release()).At(input_timestamp1)));
|
||||
|
||||
auto input_vector2 = absl::make_unique<std::vector<int>>();
|
||||
input_vector2->emplace_back(2);
|
||||
input_vector2->emplace_back(3);
|
||||
Timestamp input_timestamp2 = Timestamp(2);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector2.release()).At(input_timestamp2)));
|
||||
|
||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
|
||||
ASSERT_EQ(2, output_packets_.size());
|
||||
|
||||
EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp());
|
||||
std::vector<int> expected_output_vector0 = {1, 2};
|
||||
EXPECT_EQ(expected_output_vector0,
|
||||
output_packets_[0].Get<std::vector<int>>());
|
||||
|
||||
// At input_timestamp1, EndLoopCalc will forward timestamp bound as there are
|
||||
// no elements in vector to process.
|
||||
|
||||
EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp());
|
||||
std::vector<int> expected_output_vector2 = {3, 4};
|
||||
EXPECT_EQ(expected_output_vector2,
|
||||
output_packets_[1].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
class MultiplierCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<int>();
|
||||
cc->Inputs().Index(1).Set<int>();
|
||||
cc->Outputs().Index(0).Set<int>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
const int& input_int = cc->Inputs().Index(0).Get<int>();
|
||||
const int& multiplier_int = cc->Inputs().Index(1).Get<int>();
|
||||
auto output_int = absl::make_unique<int>(input_int * multiplier_int);
|
||||
cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(MultiplierCalculator);
|
||||
|
||||
class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
|
||||
protected:
|
||||
BeginEndLoopCalculatorGraphWithClonedInputsTest() {
|
||||
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
num_threads: 4
|
||||
input_stream: "ints"
|
||||
input_stream: "multiplier"
|
||||
node {
|
||||
calculator: "BeginLoopIntegerCalculator"
|
||||
input_stream: "ITERABLE:ints"
|
||||
input_stream: "CLONE:multiplier"
|
||||
output_stream: "ITEM:int_at_loop"
|
||||
output_stream: "CLONE:multiplier_cloned_at_loop"
|
||||
output_stream: "BATCH_END:timestamp"
|
||||
}
|
||||
node {
|
||||
calculator: "MultiplierCalculator"
|
||||
input_stream: "int_at_loop"
|
||||
input_stream: "multiplier_cloned_at_loop"
|
||||
output_stream: "multiplied_int_at_loop"
|
||||
}
|
||||
node {
|
||||
calculator: "EndLoopIntegersCalculator"
|
||||
input_stream: "ITEM:multiplied_int_at_loop"
|
||||
input_stream: "BATCH_END:timestamp"
|
||||
output_stream: "ITERABLE:multiplied_ints"
|
||||
}
|
||||
)");
|
||||
tool::AddVectorSink("multiplied_ints", &graph_config_, &output_packets_);
|
||||
}
|
||||
|
||||
CalculatorGraphConfig graph_config_;
|
||||
std::vector<Packet> output_packets_;
|
||||
};
|
||||
|
||||
TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleEmptyVector) {
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config_));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
auto input_vector = absl::make_unique<std::vector<int>>();
|
||||
Timestamp input_timestamp = Timestamp(42);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector.release()).At(input_timestamp)));
|
||||
auto multiplier = absl::make_unique<int>(2);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"multiplier", Adopt(multiplier.release()).At(input_timestamp)));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
// EndLoopCalc will forward the timestamp bound because there are no elements
|
||||
// in collection to output.
|
||||
ASSERT_EQ(0, output_packets_.size());
|
||||
|
||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleNonEmptyVector) {
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config_));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
auto input_vector = absl::make_unique<std::vector<int>>();
|
||||
input_vector->emplace_back(0);
|
||||
input_vector->emplace_back(1);
|
||||
input_vector->emplace_back(2);
|
||||
Timestamp input_timestamp = Timestamp(42);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector.release()).At(input_timestamp)));
|
||||
auto multiplier = absl::make_unique<int>(2);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"multiplier", Adopt(multiplier.release()).At(input_timestamp)));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
ASSERT_EQ(1, output_packets_.size());
|
||||
EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp());
|
||||
std::vector<int> expected_output_vector = {0, 2, 4};
|
||||
EXPECT_EQ(expected_output_vector, output_packets_[0].Get<std::vector<int>>());
|
||||
|
||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, MultipleVectors) {
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config_));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
|
||||
auto input_vector0 = absl::make_unique<std::vector<int>>();
|
||||
input_vector0->emplace_back(0);
|
||||
input_vector0->emplace_back(1);
|
||||
Timestamp input_timestamp0 = Timestamp(42);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector0.release()).At(input_timestamp0)));
|
||||
auto multiplier0 = absl::make_unique<int>(2);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"multiplier", Adopt(multiplier0.release()).At(input_timestamp0)));
|
||||
|
||||
auto input_vector1 = absl::make_unique<std::vector<int>>();
|
||||
Timestamp input_timestamp1 = Timestamp(43);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector1.release()).At(input_timestamp1)));
|
||||
auto multiplier1 = absl::make_unique<int>(2);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"multiplier", Adopt(multiplier1.release()).At(input_timestamp1)));
|
||||
|
||||
auto input_vector2 = absl::make_unique<std::vector<int>>();
|
||||
input_vector2->emplace_back(2);
|
||||
input_vector2->emplace_back(3);
|
||||
Timestamp input_timestamp2 = Timestamp(44);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"ints", Adopt(input_vector2.release()).At(input_timestamp2)));
|
||||
auto multiplier2 = absl::make_unique<int>(3);
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"multiplier", Adopt(multiplier2.release()).At(input_timestamp2)));
|
||||
|
||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
|
||||
ASSERT_EQ(2, output_packets_.size());
|
||||
|
||||
EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp());
|
||||
std::vector<int> expected_output_vector0 = {0, 2};
|
||||
EXPECT_EQ(expected_output_vector0,
|
||||
output_packets_[0].Get<std::vector<int>>());
|
||||
|
||||
// At input_timestamp1, EndLoopCalc will forward timestamp bound as there are
|
||||
// no elements in vector to process.
|
||||
|
||||
EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp());
|
||||
std::vector<int> expected_output_vector2 = {6, 9};
|
||||
EXPECT_EQ(expected_output_vector2,
|
||||
output_packets_[1].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,34 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/begin_loop_calculator.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// A calculator to process std::vector<NormalizedLandmarkList>.
|
||||
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmarkList>>
|
||||
BeginLoopNormalizedLandmarkListVectorCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkListVectorCalculator);
|
||||
|
||||
// A calculator to process std::vector<NormalizedRect>.
|
||||
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
|
||||
BeginLoopNormalizedRectCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopNormalizedRectCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,157 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Calculator for implementing loops on iterable collections inside a MediaPipe
|
||||
// graph.
|
||||
//
|
||||
// It is designed to be used like:
|
||||
//
|
||||
// node {
|
||||
// calculator: "BeginLoopWithIterableCalculator"
|
||||
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
|
||||
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
|
||||
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "ElementToBlaConverterSubgraph"
|
||||
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
|
||||
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "EndLoopWithOutputCalculator"
|
||||
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// output_stream: "OUTPUT:aggregated_result" # IterableU @ext_ts
|
||||
// }
|
||||
//
|
||||
// BeginLoopCalculator accepts an optional input stream tagged with "TICK"
|
||||
// which if non-empty, wakes up the calculator and calls
|
||||
// BeginLoopCalculator::Process(). Input streams tagged with "CLONE" are cloned
|
||||
// to the corresponding output streams at loop timestamps. This ensures that a
|
||||
// MediaPipe graph or sub-graph can run multiple times, once per element in the
|
||||
// "ITERABLE" for each pakcet clone of the packets in the "CLONE" input streams.
|
||||
template <typename IterableT>
|
||||
class BeginLoopCalculator : public CalculatorBase {
|
||||
using ItemT = typename IterableT::value_type;
|
||||
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
// A non-empty packet in the optional "TICK" input stream wakes up the
|
||||
// calculator.
|
||||
if (cc->Inputs().HasTag("TICK")) {
|
||||
cc->Inputs().Tag("TICK").SetAny();
|
||||
}
|
||||
|
||||
// An iterable collection in the input stream.
|
||||
RET_CHECK(cc->Inputs().HasTag("ITERABLE"));
|
||||
cc->Inputs().Tag("ITERABLE").Set<IterableT>();
|
||||
|
||||
// An element from the collection.
|
||||
RET_CHECK(cc->Outputs().HasTag("ITEM"));
|
||||
cc->Outputs().Tag("ITEM").Set<ItemT>();
|
||||
|
||||
RET_CHECK(cc->Outputs().HasTag("BATCH_END"));
|
||||
cc->Outputs()
|
||||
.Tag("BATCH_END")
|
||||
.Set<Timestamp>(
|
||||
// A flush signal to the corresponding EndLoopCalculator for it to
|
||||
// emit the aggregated result with the timestamp contained in this
|
||||
// flush signal packet.
|
||||
);
|
||||
|
||||
// Input streams tagged with "CLONE" are cloned to the corresponding
|
||||
// "CLONE" output streams at loop timestamps.
|
||||
RET_CHECK(cc->Inputs().NumEntries("CLONE") ==
|
||||
cc->Outputs().NumEntries("CLONE"));
|
||||
if (cc->Inputs().NumEntries("CLONE") > 0) {
|
||||
for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) {
|
||||
cc->Inputs().Get("CLONE", i).SetAny();
|
||||
cc->Outputs().Get("CLONE", i).SetSameAs(&cc->Inputs().Get("CLONE", i));
|
||||
}
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) final {
|
||||
Timestamp last_timestamp = loop_internal_timestamp_;
|
||||
if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) {
|
||||
const IterableT& collection =
|
||||
cc->Inputs().Tag("ITERABLE").template Get<IterableT>();
|
||||
for (const auto& item : collection) {
|
||||
cc->Outputs().Tag("ITEM").AddPacket(
|
||||
MakePacket<ItemT>(item).At(loop_internal_timestamp_));
|
||||
ForwardClonePackets(cc, loop_internal_timestamp_);
|
||||
++loop_internal_timestamp_;
|
||||
}
|
||||
}
|
||||
|
||||
// The collection was empty and nothing was processed.
|
||||
if (last_timestamp == loop_internal_timestamp_) {
|
||||
// Increment loop_internal_timestamp_ because it is used up now.
|
||||
++loop_internal_timestamp_;
|
||||
for (auto it = cc->Outputs().begin(); it < cc->Outputs().end(); ++it) {
|
||||
it->SetNextTimestampBound(loop_internal_timestamp_);
|
||||
}
|
||||
}
|
||||
|
||||
// The for loop processing the input collection already incremented
|
||||
// loop_internal_timestamp_. To emit BATCH_END packet along the last
|
||||
// non-BATCH_END packet, decrement by one.
|
||||
cc->Outputs()
|
||||
.Tag("BATCH_END")
|
||||
.AddPacket(MakePacket<Timestamp>(cc->InputTimestamp())
|
||||
.At(Timestamp(loop_internal_timestamp_ - 1)));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
void ForwardClonePackets(CalculatorContext* cc, Timestamp output_timestamp) {
|
||||
if (cc->Inputs().NumEntries("CLONE") > 0) {
|
||||
for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) {
|
||||
if (!cc->Inputs().Get("CLONE", i).IsEmpty()) {
|
||||
auto input_packet = cc->Inputs().Get("CLONE", i).Value();
|
||||
cc->Outputs()
|
||||
.Get("CLONE", i)
|
||||
.AddPacket(std::move(input_packet).At(output_timestamp));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fake timestamps generated per element in collection.
|
||||
Timestamp loop_internal_timestamp_ = Timestamp(0);
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
typedef ClipVectorSizeCalculator<::mediapipe::Detection>
|
||||
ClipDetectionVectorSizeCalculator;
|
||||
REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
typedef ClipVectorSizeCalculator<::mediapipe::NormalizedRect>
|
||||
ClipNormalizedRectVectorSizeCalculator;
|
||||
REGISTER_CALCULATOR(ClipNormalizedRectVectorSizeCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,148 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
|
||||
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/core/clip_vector_size_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Clips the size of the input vector of type T to a specified max_vec_size.
|
||||
// In a graph it will be used as:
|
||||
// node {
|
||||
// calculator: "ClipIntVectorSizeCalculator"
|
||||
// input_stream: "input_vector"
|
||||
// output_stream: "output_vector"
|
||||
// options {
|
||||
// [mediapipe.ClipIntVectorSizeCalculatorOptions.ext] {
|
||||
// max_vec_size: 5
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// Optionally, you can pass in a side packet that will override `max_vec_size`
|
||||
// that is specified in the options.
|
||||
template <typename T>
|
||||
class ClipVectorSizeCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().NumEntries() == 1);
|
||||
RET_CHECK(cc->Outputs().NumEntries() == 1);
|
||||
|
||||
if (cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
|
||||
.max_vec_size() < 1) {
|
||||
return ::mediapipe::InternalError(
|
||||
"max_vec_size should be greater than or equal to 1.");
|
||||
}
|
||||
|
||||
cc->Inputs().Index(0).Set<std::vector<T>>();
|
||||
cc->Outputs().Index(0).Set<std::vector<T>>();
|
||||
// Optional input side packet that determines `max_vec_size`.
|
||||
if (cc->InputSidePackets().NumEntries() > 0) {
|
||||
cc->InputSidePackets().Index(0).Set<int>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
|
||||
.max_vec_size();
|
||||
// Override `max_vec_size` if passed as side packet.
|
||||
if (cc->InputSidePackets().NumEntries() > 0 &&
|
||||
!cc->InputSidePackets().Index(0).IsEmpty()) {
|
||||
max_vec_size_ = cc->InputSidePackets().Index(0).Get<int>();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (max_vec_size_ < 1) {
|
||||
return ::mediapipe::InternalError(
|
||||
"max_vec_size should be greater than or equal to 1.");
|
||||
}
|
||||
if (cc->Inputs().Index(0).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
return ClipVectorSize<T>(std::is_copy_constructible<T>(), cc);
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
::mediapipe::Status ClipVectorSize(std::true_type, CalculatorContext* cc) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
const std::vector<U>& input_vector =
|
||||
cc->Inputs().Index(0).Get<std::vector<U>>();
|
||||
if (max_vec_size_ >= input_vector.size()) {
|
||||
output->insert(output->end(), input_vector.begin(), input_vector.end());
|
||||
} else {
|
||||
for (int i = 0; i < max_vec_size_; ++i) {
|
||||
output->push_back(input_vector[i]);
|
||||
}
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
::mediapipe::Status ClipVectorSize(std::false_type, CalculatorContext* cc) {
|
||||
return ConsumeAndClipVectorSize<T>(std::is_move_constructible<U>(), cc);
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
::mediapipe::Status ConsumeAndClipVectorSize(std::true_type,
|
||||
CalculatorContext* cc) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
|
||||
cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
|
||||
|
||||
if (input_status.ok()) {
|
||||
std::unique_ptr<std::vector<U>> input_vector =
|
||||
std::move(input_status).ValueOrDie();
|
||||
auto begin_it = input_vector->begin();
|
||||
auto end_it = input_vector->end();
|
||||
if (max_vec_size_ < input_vector->size()) {
|
||||
end_it = input_vector->begin() + max_vec_size_;
|
||||
}
|
||||
output->insert(output->end(), std::make_move_iterator(begin_it),
|
||||
std::make_move_iterator(end_it));
|
||||
} else {
|
||||
return input_status.status();
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
::mediapipe::Status ConsumeAndClipVectorSize(std::false_type,
|
||||
CalculatorContext* cc) {
|
||||
return ::mediapipe::InternalError(
|
||||
"Cannot copy or move input vectors and clip their size.");
|
||||
}
|
||||
|
||||
private:
|
||||
int max_vec_size_ = 0;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message ClipVectorSizeCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional ClipVectorSizeCalculatorOptions ext = 274674998;
|
||||
}
|
||||
|
||||
// Maximum size of output vector.
|
||||
optional int32 max_vec_size = 1 [default = 1];
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
typedef ClipVectorSizeCalculator<int> TestClipIntVectorSizeCalculator;
|
||||
REGISTER_CALCULATOR(TestClipIntVectorSizeCalculator);
|
||||
|
||||
void AddInputVector(const std::vector<int>& input, int64 timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
runner->MutableInputs()->Index(0).packets.push_back(
|
||||
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
|
||||
}
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
|
||||
std::vector<int> input = {};
|
||||
AddInputVector(input, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
EXPECT_TRUE(outputs[0].Get<std::vector<int>>().empty());
|
||||
}
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 }
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
|
||||
std::vector<int> input = {0, 1, 2, 3};
|
||||
AddInputVector(input, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
|
||||
EXPECT_EQ(2, output.size());
|
||||
std::vector<int> expected_vector = {0, 1};
|
||||
EXPECT_EQ(expected_vector, output);
|
||||
}
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
|
||||
{
|
||||
std::vector<int> input = {0, 1, 2, 3};
|
||||
AddInputVector(input, /*timestamp=*/1, &runner);
|
||||
}
|
||||
{
|
||||
std::vector<int> input = {2, 3, 4, 5};
|
||||
AddInputVector(input, /*timestamp=*/2, &runner);
|
||||
}
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(2, outputs.size());
|
||||
{
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
|
||||
EXPECT_EQ(3, output.size());
|
||||
std::vector<int> expected_vector = {0, 1, 2};
|
||||
EXPECT_EQ(expected_vector, output);
|
||||
}
|
||||
{
|
||||
EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
|
||||
const std::vector<int>& output = outputs[1].Get<std::vector<int>>();
|
||||
EXPECT_EQ(3, output.size());
|
||||
std::vector<int> expected_vector = {2, 3, 4};
|
||||
EXPECT_EQ(expected_vector, output);
|
||||
}
|
||||
}
|
||||
|
||||
typedef ClipVectorSizeCalculator<std::unique_ptr<int>>
|
||||
TestClipUniqueIntPtrVectorSizeCalculator;
|
||||
REGISTER_CALCULATOR(TestClipUniqueIntPtrVectorSizeCalculator);
|
||||
|
||||
TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
|
||||
/* Note: We don't use CalculatorRunner for this test because it keeps copies
|
||||
* of input packets, so packets sent to the graph don't have sole ownership.
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "TestClipUniqueIntPtrVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("output_vector", &graph_config, &outputs);
|
||||
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
|
||||
// input1 : {0, 1, 2, 3, 4, 5}
|
||||
auto input_vector = absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
|
||||
for (int i = 0; i < 6; ++i) {
|
||||
input_vector->at(i) = absl::make_unique<int>(i);
|
||||
}
|
||||
|
||||
MP_EXPECT_OK(graph.AddPacketToInputStream(
|
||||
"input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
|
||||
|
||||
MP_EXPECT_OK(graph.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph.CloseAllPacketSources());
|
||||
MP_EXPECT_OK(graph.WaitUntilDone());
|
||||
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
const std::vector<std::unique_ptr<int>>& result =
|
||||
outputs[0].Get<std::vector<std::unique_ptr<int>>>();
|
||||
EXPECT_EQ(3, result.size());
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
const std::unique_ptr<int>& v = result[i];
|
||||
EXPECT_EQ(i, *v);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
input_side_packet: "max_vec_size"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
// This should override the default of 1 set in the options.
|
||||
runner.MutableSidePackets()->Index(0) = Adopt(new int(2));
|
||||
std::vector<int> input = {0, 1, 2, 3};
|
||||
AddInputVector(input, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
|
||||
EXPECT_EQ(2, output.size());
|
||||
std::vector<int> expected_vector = {0, 1};
|
||||
EXPECT_EQ(expected_vector, output);
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/core/concatenate_vector_calculator.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "ConcatenateDetectionVectorCalculator"
|
||||
// input_stream: "detection_vector_1"
|
||||
// input_stream: "detection_vector_2"
|
||||
// output_stream: "concatenated_detection_vector"
|
||||
// }
|
||||
typedef ConcatenateVectorCalculator<::mediapipe::Detection>
|
||||
ConcatenateDetectionVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateDetectionVectorCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/concatenate_vector_calculator.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "ConcatenateFloatVectorCalculator"
|
||||
// input_stream: "float_vector_1"
|
||||
// input_stream: "float_vector_2"
|
||||
// output_stream: "concatenated_float_vector"
|
||||
// }
|
||||
typedef ConcatenateVectorCalculator<float> ConcatenateFloatVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateFloatVectorCalculator);
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "ConcatenateInt32VectorCalculator"
|
||||
// input_stream: "int32_vector_1"
|
||||
// input_stream: "int32_vector_2"
|
||||
// output_stream: "concatenated_int32_vector"
|
||||
// }
|
||||
typedef ConcatenateVectorCalculator<int32> ConcatenateInt32VectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
|
||||
// input_stream: "tflitetensor_vector_1"
|
||||
// input_stream: "tflitetensor_vector_2"
|
||||
// output_stream: "concatenated_tflitetensor_vector"
|
||||
// }
|
||||
typedef ConcatenateVectorCalculator<TfLiteTensor>
|
||||
ConcatenateTfLiteTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
|
||||
ConcatenateLandmarkVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
|
||||
ConcatenateGlBufferVectorCalculator;
|
||||
REGISTER_CALCULATOR(ConcatenateGlBufferVectorCalculator);
|
||||
#endif
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,121 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
|
||||
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Concatenates several std::vector<T> following stream index order. This class
|
||||
// assumes that every input stream contains the vector<T> type. To use this
|
||||
// class for a particular type T, regisiter a calculator using
|
||||
// ConcatenateVectorCalculator<T>.
|
||||
template <typename T>
|
||||
class ConcatenateVectorCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().NumEntries() != 0);
|
||||
RET_CHECK(cc->Outputs().NumEntries() == 1);
|
||||
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
cc->Inputs().Index(i).Set<std::vector<T>>();
|
||||
}
|
||||
|
||||
cc->Outputs().Index(0).Set<std::vector<T>>();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
only_emit_if_all_present_ =
|
||||
cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
|
||||
.only_emit_if_all_present();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (only_emit_if_all_present_) {
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (cc->Inputs().Index(i).IsEmpty()) return ::mediapipe::OkStatus();
|
||||
}
|
||||
}
|
||||
|
||||
return ConcatenateVectors<T>(std::is_copy_constructible<T>(), cc);
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
::mediapipe::Status ConcatenateVectors(std::true_type,
|
||||
CalculatorContext* cc) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (cc->Inputs().Index(i).IsEmpty()) continue;
|
||||
const std::vector<U>& input = cc->Inputs().Index(i).Get<std::vector<U>>();
|
||||
output->insert(output->end(), input.begin(), input.end());
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
::mediapipe::Status ConcatenateVectors(std::false_type,
|
||||
CalculatorContext* cc) {
|
||||
return ConsumeAndConcatenateVectors<T>(std::is_move_constructible<U>(), cc);
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
::mediapipe::Status ConsumeAndConcatenateVectors(std::true_type,
|
||||
CalculatorContext* cc) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (cc->Inputs().Index(i).IsEmpty()) continue;
|
||||
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
|
||||
cc->Inputs().Index(i).Value().Consume<std::vector<U>>();
|
||||
if (input_status.ok()) {
|
||||
std::unique_ptr<std::vector<U>> input_vector =
|
||||
std::move(input_status).ValueOrDie();
|
||||
output->insert(output->end(),
|
||||
std::make_move_iterator(input_vector->begin()),
|
||||
std::make_move_iterator(input_vector->end()));
|
||||
} else {
|
||||
return input_status.status();
|
||||
}
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
::mediapipe::Status ConsumeAndConcatenateVectors(std::false_type,
|
||||
CalculatorContext* cc) {
|
||||
return ::mediapipe::InternalError(
|
||||
"Cannot copy or move input vectors to concatenate them");
|
||||
}
|
||||
|
||||
private:
|
||||
bool only_emit_if_all_present_;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
|
||||
@@ -0,0 +1,29 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message ConcatenateVectorCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional ConcatenateVectorCalculatorOptions ext = 259397839;
|
||||
}
|
||||
|
||||
// If true, the calculator will only emit a packet at the given timestamp if
|
||||
// all input streams have a non-empty packet (AND operation on streams).
|
||||
optional bool only_emit_if_all_present = 1 [default = false];
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/concatenate_vector_calculator.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
|
||||
REGISTER_CALCULATOR(TestConcatenateIntVectorCalculator);
|
||||
|
||||
void AddInputVectors(const std::vector<std::vector<int>>& inputs,
|
||||
int64 timestamp, CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
runner->MutableInputs()->Index(i).packets.push_back(
|
||||
MakePacket<std::vector<int>>(inputs[i]).At(Timestamp(timestamp)));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, EmptyVectorInputs) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<int>> inputs = {{}, {}, {}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_TRUE(outputs[0].Get<std::vector<int>>().empty());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, OneTimestamp) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<int>> inputs = {{1, 2, 3}, {4}, {5, 6}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3, 4, 5, 6};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, TwoInputsAtTwoTimestamps) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
{
|
||||
std::vector<std::vector<int>> inputs = {{1, 2, 3}, {4}, {5, 6}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
}
|
||||
{
|
||||
std::vector<std::vector<int>> inputs = {{0, 2}, {1}, {3, 5}};
|
||||
AddInputVectors(inputs, /*timestamp=*/2, &runner);
|
||||
}
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(2, outputs.size());
|
||||
{
|
||||
EXPECT_EQ(6, outputs[0].Get<std::vector<int>>().size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3, 4, 5, 6};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
{
|
||||
EXPECT_EQ(5, outputs[1].Get<std::vector<int>>().size());
|
||||
EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
|
||||
std::vector<int> expected_vector = {0, 2, 1, 3, 5};
|
||||
EXPECT_EQ(expected_vector, outputs[1].Get<std::vector<int>>());
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/2,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<int>> inputs = {{1, 2, 3}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<int> expected_vector = {1, 2, 3};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
|
||||
}
|
||||
|
||||
TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
CalculatorRunner runner("TestConcatenateIntVectorCalculator",
|
||||
/*options_string=*/
|
||||
"[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
|
||||
"{only_emit_if_all_present: true}",
|
||||
/*num_inputs=*/2,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<int>> inputs = {{1, 2, 3}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
void AddInputVectors(const std::vector<std::vector<float>>& inputs,
|
||||
int64 timestamp, CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
runner->MutableInputs()->Index(i).packets.push_back(
|
||||
MakePacket<std::vector<float>>(inputs[i]).At(Timestamp(timestamp)));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ConcatenateFloatVectorCalculatorTest, EmptyVectorInputs) {
|
||||
CalculatorRunner runner("ConcatenateFloatVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<float>> inputs = {{}, {}, {}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_TRUE(outputs[0].Get<std::vector<float>>().empty());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
}
|
||||
|
||||
TEST(ConcatenateFloatVectorCalculatorTest, OneTimestamp) {
|
||||
CalculatorRunner runner("ConcatenateFloatVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<float>> inputs = {
|
||||
{1.0f, 2.0f, 3.0f}, {4.0f}, {5.0f, 6.0f}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<float> expected_vector = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<float>>());
|
||||
}
|
||||
|
||||
TEST(ConcatenateFloatVectorCalculatorTest, TwoInputsAtTwoTimestamps) {
|
||||
CalculatorRunner runner("ConcatenateFloatVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
{
|
||||
std::vector<std::vector<float>> inputs = {
|
||||
{1.0f, 2.0f, 3.0f}, {4.0f}, {5.0f, 6.0f}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
}
|
||||
{
|
||||
std::vector<std::vector<float>> inputs = {
|
||||
{0.0f, 2.0f}, {1.0f}, {3.0f, 5.0f}};
|
||||
AddInputVectors(inputs, /*timestamp=*/2, &runner);
|
||||
}
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(2, outputs.size());
|
||||
{
|
||||
EXPECT_EQ(6, outputs[0].Get<std::vector<float>>().size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<float> expected_vector = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<float>>());
|
||||
}
|
||||
{
|
||||
EXPECT_EQ(5, outputs[1].Get<std::vector<float>>().size());
|
||||
EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
|
||||
std::vector<float> expected_vector = {0.0f, 2.0f, 1.0f, 3.0f, 5.0f};
|
||||
EXPECT_EQ(expected_vector, outputs[1].Get<std::vector<float>>());
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamStillOutput) {
|
||||
CalculatorRunner runner("ConcatenateFloatVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/2,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<float>> inputs = {{1.0f, 2.0f, 3.0f}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<float> expected_vector = {1.0f, 2.0f, 3.0f};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<float>>());
|
||||
}
|
||||
|
||||
TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
CalculatorRunner runner("ConcatenateFloatVectorCalculator",
|
||||
/*options_string=*/
|
||||
"[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
|
||||
"{only_emit_if_all_present: true}",
|
||||
/*num_inputs=*/2,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<float>> inputs = {{1.0f, 2.0f, 3.0f}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
|
||||
TestConcatenateUniqueIntPtrCalculator;
|
||||
REGISTER_CALCULATOR(TestConcatenateUniqueIntPtrCalculator);
|
||||
|
||||
TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
|
||||
/* Note: We don't use CalculatorRunner for this test because it keeps copies
|
||||
* of input packets, so packets sent to the graph don't have sole ownership.
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
input_stream: "in_3"
|
||||
node {
|
||||
calculator: "TestConcatenateUniqueIntPtrCalculator"
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
input_stream: "in_3"
|
||||
output_stream: "out"
|
||||
}
|
||||
)");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("out", &graph_config, &outputs);
|
||||
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
|
||||
// input1 : {0, 1, 2}
|
||||
std::unique_ptr<std::vector<std::unique_ptr<int>>> input_1 =
|
||||
absl::make_unique<std::vector<std::unique_ptr<int>>>(3);
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
input_1->at(i) = absl::make_unique<int>(i);
|
||||
}
|
||||
// input2: {3}
|
||||
std::unique_ptr<std::vector<std::unique_ptr<int>>> input_2 =
|
||||
absl::make_unique<std::vector<std::unique_ptr<int>>>(1);
|
||||
input_2->at(0) = absl::make_unique<int>(3);
|
||||
// input3: {4, 5}
|
||||
std::unique_ptr<std::vector<std::unique_ptr<int>>> input_3 =
|
||||
absl::make_unique<std::vector<std::unique_ptr<int>>>(2);
|
||||
input_3->at(0) = absl::make_unique<int>(4);
|
||||
input_3->at(1) = absl::make_unique<int>(5);
|
||||
|
||||
MP_EXPECT_OK(graph.AddPacketToInputStream(
|
||||
"in_1", Adopt(input_1.release()).At(Timestamp(1))));
|
||||
MP_EXPECT_OK(graph.AddPacketToInputStream(
|
||||
"in_2", Adopt(input_2.release()).At(Timestamp(1))));
|
||||
MP_EXPECT_OK(graph.AddPacketToInputStream(
|
||||
"in_3", Adopt(input_3.release()).At(Timestamp(1))));
|
||||
|
||||
MP_EXPECT_OK(graph.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph.CloseAllPacketSources());
|
||||
MP_EXPECT_OK(graph.WaitUntilDone());
|
||||
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
const std::vector<std::unique_ptr<int>>& result =
|
||||
outputs[0].Get<std::vector<std::unique_ptr<int>>>();
|
||||
EXPECT_EQ(6, result.size());
|
||||
for (int i = 0; i < 6; ++i) {
|
||||
const std::unique_ptr<int>& v = result[i];
|
||||
EXPECT_EQ(i, *v);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
|
||||
/* Note: We don't use CalculatorRunner for this test because it keeps copies
|
||||
* of input packets, so packets sent to the graph don't have sole ownership.
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
node {
|
||||
calculator: "TestConcatenateUniqueIntPtrCalculator"
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
output_stream: "out"
|
||||
}
|
||||
)");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("out", &graph_config, &outputs);
|
||||
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
|
||||
// input1 : {0, 1, 2}
|
||||
std::unique_ptr<std::vector<std::unique_ptr<int>>> input_1 =
|
||||
absl::make_unique<std::vector<std::unique_ptr<int>>>(3);
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
input_1->at(i) = absl::make_unique<int>(i);
|
||||
}
|
||||
|
||||
MP_EXPECT_OK(graph.AddPacketToInputStream(
|
||||
"in_1", Adopt(input_1.release()).At(Timestamp(1))));
|
||||
|
||||
MP_EXPECT_OK(graph.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph.CloseAllPacketSources());
|
||||
MP_EXPECT_OK(graph.WaitUntilDone());
|
||||
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
const std::vector<std::unique_ptr<int>>& result =
|
||||
outputs[0].Get<std::vector<std::unique_ptr<int>>>();
|
||||
EXPECT_EQ(3, result.size());
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
const std::unique_ptr<int>& v = result[i];
|
||||
EXPECT_EQ(i, *v);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
/* Note: We don't use CalculatorRunner for this test because it keeps copies
|
||||
* of input packets, so packets sent to the graph don't have sole ownership.
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
node {
|
||||
calculator: "TestConcatenateUniqueIntPtrCalculator"
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
output_stream: "out"
|
||||
options {
|
||||
[mediapipe.ConcatenateVectorCalculatorOptions.ext] {
|
||||
only_emit_if_all_present: true
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("out", &graph_config, &outputs);
|
||||
|
||||
CalculatorGraph graph;
|
||||
MP_EXPECT_OK(graph.Initialize(graph_config));
|
||||
MP_EXPECT_OK(graph.StartRun({}));
|
||||
|
||||
// input1 : {0, 1, 2}
|
||||
std::unique_ptr<std::vector<std::unique_ptr<int>>> input_1 =
|
||||
absl::make_unique<std::vector<std::unique_ptr<int>>>(3);
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
input_1->at(i) = absl::make_unique<int>(i);
|
||||
}
|
||||
|
||||
MP_EXPECT_OK(graph.AddPacketToInputStream(
|
||||
"in_1", Adopt(input_1.release()).At(Timestamp(1))));
|
||||
|
||||
MP_EXPECT_OK(graph.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph.CloseAllPacketSources());
|
||||
MP_EXPECT_OK(graph.WaitUntilDone());
|
||||
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,116 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "mediapipe/calculators/core/constant_side_packet_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Generates an output side packet or multiple output side packets according to
|
||||
// the specified options.
|
||||
//
|
||||
// Example configs:
|
||||
// node {
|
||||
// calculator: "ConstantSidePacketCalculator"
|
||||
// output_side_packet: "PACKET:packet"
|
||||
// options: {
|
||||
// [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
|
||||
// packet { int_value: 2 }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "ConstantSidePacketCalculator"
|
||||
// output_side_packet: "PACKET:0:int_packet"
|
||||
// output_side_packet: "PACKET:1:bool_packet"
|
||||
// options: {
|
||||
// [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
|
||||
// packet { int_value: 2 }
|
||||
// packet { bool_value: true }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class ConstantSidePacketCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
const auto& options = cc->Options().GetExtension(
|
||||
::mediapipe::ConstantSidePacketCalculatorOptions::ext);
|
||||
RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag),
|
||||
options.packet_size())
|
||||
<< "Number of output side packets has to be same as number of packets "
|
||||
"configured in options.";
|
||||
|
||||
int index = 0;
|
||||
for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
|
||||
id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {
|
||||
const auto& packet_options = options.packet(index);
|
||||
auto& packet = cc->OutputSidePackets().Get(id);
|
||||
if (packet_options.has_int_value()) {
|
||||
packet.Set<int>();
|
||||
} else if (packet_options.has_float_value()) {
|
||||
packet.Set<float>();
|
||||
} else if (packet_options.has_bool_value()) {
|
||||
packet.Set<bool>();
|
||||
} else if (packet_options.has_string_value()) {
|
||||
packet.Set<std::string>();
|
||||
} else {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"None of supported values were specified in options.");
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
const auto& options = cc->Options().GetExtension(
|
||||
::mediapipe::ConstantSidePacketCalculatorOptions::ext);
|
||||
int index = 0;
|
||||
for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
|
||||
id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {
|
||||
auto& packet = cc->OutputSidePackets().Get(id);
|
||||
const auto& packet_options = options.packet(index);
|
||||
if (packet_options.has_int_value()) {
|
||||
packet.Set(MakePacket<int>(packet_options.int_value()));
|
||||
} else if (packet_options.has_float_value()) {
|
||||
packet.Set(MakePacket<float>(packet_options.float_value()));
|
||||
} else if (packet_options.has_bool_value()) {
|
||||
packet.Set(MakePacket<bool>(packet_options.bool_value()));
|
||||
} else if (packet_options.has_string_value()) {
|
||||
packet.Set(MakePacket<std::string>(packet_options.string_value()));
|
||||
} else {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"None of supported values were specified in options.");
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr const char* kPacketTag = "PACKET";
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(ConstantSidePacketCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message ConstantSidePacketCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional ConstantSidePacketCalculatorOptions ext = 291214597;
|
||||
}
|
||||
|
||||
message ConstantSidePacket {
|
||||
oneof value {
|
||||
int32 int_value = 1;
|
||||
float float_value = 2;
|
||||
bool bool_value = 3;
|
||||
string string_value = 4;
|
||||
}
|
||||
}
|
||||
|
||||
repeated ConstantSidePacket packet = 1;
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
template <typename T>
|
||||
void DoTestSingleSidePacket(absl::string_view packet_spec,
|
||||
const T& expected_value) {
|
||||
static constexpr absl::string_view graph_config_template = R"(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:packet"
|
||||
options: {
|
||||
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
|
||||
packet $0
|
||||
}
|
||||
}
|
||||
}
|
||||
)";
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
absl::Substitute(graph_config_template, packet_spec));
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("packet"));
|
||||
auto actual_value =
|
||||
graph.GetOutputSidePacket("packet").ValueOrDie().template Get<T>();
|
||||
EXPECT_EQ(actual_value, expected_value);
|
||||
}
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
|
||||
DoTestSingleSidePacket("{ int_value: 2 }", 2);
|
||||
DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
|
||||
DoTestSingleSidePacket("{ bool_value: true }", true);
|
||||
DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
|
||||
}
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:0:int_packet"
|
||||
output_side_packet: "PACKET:1:float_packet"
|
||||
output_side_packet: "PACKET:2:bool_packet"
|
||||
output_side_packet: "PACKET:3:string_packet"
|
||||
output_side_packet: "PACKET:4:another_string_packet"
|
||||
output_side_packet: "PACKET:5:another_int_packet"
|
||||
options: {
|
||||
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
|
||||
packet { int_value: 256 }
|
||||
packet { float_value: 0.5f }
|
||||
packet { bool_value: false }
|
||||
packet { string_value: "string" }
|
||||
packet { string_value: "another string" }
|
||||
packet { int_value: 128 }
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet"));
|
||||
EXPECT_EQ(graph.GetOutputSidePacket("int_packet").ValueOrDie().Get<int>(),
|
||||
256);
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet"));
|
||||
EXPECT_EQ(graph.GetOutputSidePacket("float_packet").ValueOrDie().Get<float>(),
|
||||
0.5f);
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet"));
|
||||
EXPECT_FALSE(
|
||||
graph.GetOutputSidePacket("bool_packet").ValueOrDie().Get<bool>());
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet"));
|
||||
EXPECT_EQ(graph.GetOutputSidePacket("string_packet")
|
||||
.ValueOrDie()
|
||||
.Get<std::string>(),
|
||||
"string");
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("another_string_packet"));
|
||||
EXPECT_EQ(graph.GetOutputSidePacket("another_string_packet")
|
||||
.ValueOrDie()
|
||||
.Get<std::string>(),
|
||||
"another string");
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("another_int_packet"));
|
||||
EXPECT_EQ(
|
||||
graph.GetOutputSidePacket("another_int_packet").ValueOrDie().Get<int>(),
|
||||
128);
|
||||
}
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:0:int_packet"
|
||||
output_side_packet: "no_tag0"
|
||||
output_side_packet: "PACKET:1:float_packet"
|
||||
output_side_packet: "INCORRECT_TAG:0:name1"
|
||||
output_side_packet: "PACKET:2:bool_packet"
|
||||
output_side_packet: "PACKET:3:string_packet"
|
||||
output_side_packet: "no_tag2"
|
||||
output_side_packet: "INCORRECT_TAG:1:name2"
|
||||
options: {
|
||||
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
|
||||
packet { int_value: 256 }
|
||||
packet { float_value: 0.5f }
|
||||
packet { bool_value: false }
|
||||
packet { string_value: "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet"));
|
||||
EXPECT_EQ(graph.GetOutputSidePacket("int_packet").ValueOrDie().Get<int>(),
|
||||
256);
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet"));
|
||||
EXPECT_EQ(graph.GetOutputSidePacket("float_packet").ValueOrDie().Get<float>(),
|
||||
0.5f);
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet"));
|
||||
EXPECT_FALSE(
|
||||
graph.GetOutputSidePacket("bool_packet").ValueOrDie().Get<bool>());
|
||||
MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet"));
|
||||
EXPECT_EQ(graph.GetOutputSidePacket("string_packet")
|
||||
.ValueOrDie()
|
||||
.Get<std::string>(),
|
||||
"string");
|
||||
}
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:int_packet"
|
||||
options: {
|
||||
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
|
||||
packet { int_value: 256 }
|
||||
packet { float_value: 0.5f }
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorGraph graph;
|
||||
EXPECT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:0:int_packet"
|
||||
output_side_packet: "PACKET:1:float_packet"
|
||||
options: {
|
||||
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
|
||||
packet { int_value: 256 }
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorGraph graph;
|
||||
EXPECT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,90 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cfloat>
|
||||
|
||||
#include "mediapipe/calculators/core/dequantize_byte_array_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
// Dequantizes a byte array to a vector of floats.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "DequantizeByteArrayCalculator"
|
||||
// input_stream: "ENCODED:encoded"
|
||||
// output_stream: "FLOAT_VECTOR:float_vector"
|
||||
// options {
|
||||
// [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
|
||||
// max_quantized_value: 2
|
||||
// min_quantized_value: -2
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
namespace mediapipe {
|
||||
|
||||
class DequantizeByteArrayCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag("ENCODED").Set<std::string>();
|
||||
cc->Outputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) final {
|
||||
const auto options =
|
||||
cc->Options<::mediapipe::DequantizeByteArrayCalculatorOptions>();
|
||||
if (!options.has_max_quantized_value() ||
|
||||
!options.has_min_quantized_value()) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"Both max_quantized_value and min_quantized_value must be provided "
|
||||
"in DequantizeByteArrayCalculatorOptions.");
|
||||
}
|
||||
float max_quantized_value = options.max_quantized_value();
|
||||
float min_quantized_value = options.min_quantized_value();
|
||||
if (max_quantized_value < min_quantized_value + FLT_EPSILON) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"max_quantized_value must be greater than min_quantized_value.");
|
||||
}
|
||||
float range = max_quantized_value - min_quantized_value;
|
||||
scalar_ = range / 255.0;
|
||||
bias_ = (range / 512.0) + min_quantized_value;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) final {
|
||||
const std::string& encoded =
|
||||
cc->Inputs().Tag("ENCODED").Value().Get<std::string>();
|
||||
std::vector<float> float_vector;
|
||||
float_vector.reserve(encoded.length());
|
||||
for (int i = 0; i < encoded.length(); ++i) {
|
||||
float_vector.push_back(
|
||||
static_cast<unsigned char>(encoded.at(i)) * scalar_ + bias_);
|
||||
}
|
||||
cc->Outputs()
|
||||
.Tag("FLOAT_VECTOR")
|
||||
.AddPacket(MakePacket<std::vector<float>>(float_vector)
|
||||
.At(cc->InputTimestamp()));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
float scalar_;
|
||||
float bias_;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(DequantizeByteArrayCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message DequantizeByteArrayCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional DequantizeByteArrayCalculatorOptions ext = 272316343;
|
||||
}
|
||||
|
||||
optional float max_quantized_value = 1;
|
||||
optional float min_quantized_value = 2;
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
options {
|
||||
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
|
||||
max_quantized_value: 2
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
MakePacket<std::string>(empty_string).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
testing::HasSubstr(
|
||||
"Both max_quantized_value and min_quantized_value must be provided"));
|
||||
}
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
options {
|
||||
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
|
||||
max_quantized_value: -2
|
||||
min_quantized_value: 2
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
MakePacket<std::string>(empty_string).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
testing::HasSubstr(
|
||||
"max_quantized_value must be greater than min_quantized_value"));
|
||||
}
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
options {
|
||||
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
|
||||
max_quantized_value: 1
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
MakePacket<std::string>(empty_string).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
testing::HasSubstr(
|
||||
"max_quantized_value must be greater than min_quantized_value"));
|
||||
}
|
||||
|
||||
TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
options {
|
||||
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
|
||||
max_quantized_value: 2
|
||||
min_quantized_value: -2
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
MakePacket<std::string>(
|
||||
std::string(reinterpret_cast<char const*>(input), 4))
|
||||
.At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& outputs =
|
||||
runner.Outputs().Tag("FLOAT_VECTOR").packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
const std::vector<float>& result = outputs[0].Get<std::vector<float>>();
|
||||
ASSERT_FALSE(result.empty());
|
||||
EXPECT_EQ(4, result.size());
|
||||
EXPECT_NEAR(0, result[0], 0.01);
|
||||
EXPECT_NEAR(2, result[1], 0.01);
|
||||
EXPECT_NEAR(-2, result[2], 0.01);
|
||||
EXPECT_NEAR(-1.976, result[3], 0.01);
|
||||
|
||||
EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/end_loop_calculator.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/util/render_data.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
|
||||
EndLoopNormalizedRectCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopNormalizedRectCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmarkList>>
|
||||
EndLoopNormalizedLandmarkListVectorCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopNormalizedLandmarkListVectorCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<bool>> EndLoopBooleanCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopBooleanCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<::mediapipe::RenderData>>
|
||||
EndLoopRenderDataCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopRenderDataCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,106 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
|
||||
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Calculator for completing the processing of loops on iterable collections
|
||||
// inside a MediaPipe graph. The EndLoopCalculator collects all input packets
|
||||
// from ITEM input_stream into a collection and upon receiving the flush signal
|
||||
// from the "BATCH_END" tagged input stream, it emits the aggregated results
|
||||
// at the original timestamp contained in the "BATCH_END" input stream.
|
||||
//
|
||||
// It is designed to be used like:
|
||||
//
|
||||
// node {
|
||||
// calculator: "BeginLoopWithIterableCalculator"
|
||||
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
|
||||
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
|
||||
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "ElementToBlaConverterSubgraph"
|
||||
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
|
||||
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "EndLoopWithOutputCalculator"
|
||||
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// output_stream: "OUTPUT:aggregated_result" # IterableU @ext_ts
|
||||
// }
|
||||
template <typename IterableT>
|
||||
class EndLoopCalculator : public CalculatorBase {
|
||||
using ItemT = typename IterableT::value_type;
|
||||
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag("BATCH_END"))
|
||||
<< "Missing BATCH_END tagged input_stream.";
|
||||
cc->Inputs().Tag("BATCH_END").Set<Timestamp>();
|
||||
|
||||
RET_CHECK(cc->Inputs().HasTag("ITEM"));
|
||||
cc->Inputs().Tag("ITEM").Set<ItemT>();
|
||||
|
||||
RET_CHECK(cc->Outputs().HasTag("ITERABLE"));
|
||||
cc->Outputs().Tag("ITERABLE").Set<IterableT>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (!cc->Inputs().Tag("ITEM").IsEmpty()) {
|
||||
if (!input_stream_collection_) {
|
||||
input_stream_collection_.reset(new IterableT);
|
||||
}
|
||||
input_stream_collection_->push_back(
|
||||
cc->Inputs().Tag("ITEM").template Get<ItemT>());
|
||||
}
|
||||
|
||||
if (!cc->Inputs().Tag("BATCH_END").Value().IsEmpty()) { // flush signal
|
||||
Timestamp loop_control_ts =
|
||||
cc->Inputs().Tag("BATCH_END").template Get<Timestamp>();
|
||||
if (input_stream_collection_) {
|
||||
cc->Outputs()
|
||||
.Tag("ITERABLE")
|
||||
.Add(input_stream_collection_.release(), loop_control_ts);
|
||||
} else {
|
||||
// Since there is no collection, inform downstream calculators to not
|
||||
// expect any packet by updating the timestamp bounds.
|
||||
cc->Outputs()
|
||||
.Tag("ITERABLE")
|
||||
.SetNextTimestampBound(Timestamp(loop_control_ts.Value() + 1));
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<IterableT> input_stream_collection_;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
|
||||
+14
-14
@@ -23,34 +23,34 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// RealTimeFlowLimiterCalculator is used to limit the number of pipelined
|
||||
// processing operations in a section of the graph.
|
||||
// FlowLimiterCalculator is used to limit the number of pipelined processing
|
||||
// operations in a section of the graph.
|
||||
//
|
||||
// Typical topology:
|
||||
//
|
||||
// in ->-[RTFLC]-[foo]-...-[bar]-+->- out
|
||||
// ^____________________|
|
||||
// FINISHED
|
||||
// in ->-[FLC]-[foo]-...-[bar]-+->- out
|
||||
// ^_____________________|
|
||||
// FINISHED
|
||||
//
|
||||
// By connecting the output of the graph section to this calculator's FINISHED
|
||||
// input with a backwards edge, this allows RTFLC to keep track of how many
|
||||
// input with a backwards edge, this allows FLC to keep track of how many
|
||||
// timestamps are currently being processed.
|
||||
//
|
||||
// The limit defaults to 1, and can be overridden with the MAX_IN_FLIGHT side
|
||||
// packet.
|
||||
//
|
||||
// As long as the number of timestamps being processed ("in flight") is below
|
||||
// the limit, RTFLC allows input to pass through. When the limit is reached,
|
||||
// RTFLC starts dropping input packets, keeping only the most recent. When the
|
||||
// the limit, FLC allows input to pass through. When the limit is reached,
|
||||
// FLC starts dropping input packets, keeping only the most recent. When the
|
||||
// processing count decreases again, as signaled by the receipt of a packet on
|
||||
// FINISHED, RTFLC allows packets to flow again, releasing the most recently
|
||||
// FINISHED, FLC allows packets to flow again, releasing the most recently
|
||||
// queued packet, if any.
|
||||
//
|
||||
// If there are multiple input streams, packet dropping is synchronized.
|
||||
//
|
||||
// IMPORTANT: for each timestamp where RTFLC forwards a packet (or a set of
|
||||
// IMPORTANT: for each timestamp where FLC forwards a packet (or a set of
|
||||
// packets, if using multiple data streams), a packet must eventually arrive on
|
||||
// the FINISHED stream. Dropping packets in the section between RTFLC and
|
||||
// the FINISHED stream. Dropping packets in the section between FLC and
|
||||
// FINISHED will make the in-flight count incorrect.
|
||||
//
|
||||
// TODO: Remove this comment when graph-level ISH has been removed.
|
||||
@@ -61,7 +61,7 @@ namespace mediapipe {
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "RealTimeFlowLimiterCalculator"
|
||||
// calculator: "FlowLimiterCalculator"
|
||||
// input_stream: "raw_frames"
|
||||
// input_stream: "FINISHED:finished"
|
||||
// input_stream_info: {
|
||||
@@ -73,7 +73,7 @@ namespace mediapipe {
|
||||
// }
|
||||
// output_stream: "gated_frames"
|
||||
// }
|
||||
class RealTimeFlowLimiterCalculator : public CalculatorBase {
|
||||
class FlowLimiterCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
int num_data_streams = cc->Inputs().NumEntries("");
|
||||
@@ -194,6 +194,6 @@ class RealTimeFlowLimiterCalculator : public CalculatorBase {
|
||||
Timestamp allow_ctr_ts_;
|
||||
std::vector<Timestamp> data_stream_bound_ts_;
|
||||
};
|
||||
REGISTER_CALCULATOR(RealTimeFlowLimiterCalculator);
|
||||
REGISTER_CALCULATOR(FlowLimiterCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
+54
-56
@@ -71,7 +71,7 @@ constexpr int kNumImageFrames = 5;
|
||||
constexpr int kNumFinished = 3;
|
||||
CalculatorGraphConfig::Node GetDefaultNode() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "RealTimeFlowLimiterCalculator"
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "raw_frames"
|
||||
input_stream: "FINISHED:finished"
|
||||
input_stream_info: { tag_index: "FINISHED" back_edge: true }
|
||||
@@ -79,9 +79,9 @@ CalculatorGraphConfig::Node GetDefaultNode() {
|
||||
)");
|
||||
}
|
||||
|
||||
// Simple test to make sure that the RealTimeFlowLimiterCalculator outputs
|
||||
// just one packet when MAX_IN_FLIGHT is 1.
|
||||
TEST(RealTimeFlowLimiterCalculator, OneOutputTest) {
|
||||
// Simple test to make sure that the FlowLimiterCalculator outputs just one
|
||||
// packet when MAX_IN_FLIGHT is 1.
|
||||
TEST(FlowLimiterCalculator, OneOutputTest) {
|
||||
// Setup the calculator runner and add only ImageFrame packets.
|
||||
CalculatorRunner runner(GetDefaultNode());
|
||||
for (int i = 0; i < kNumImageFrames; ++i) {
|
||||
@@ -91,16 +91,16 @@ TEST(RealTimeFlowLimiterCalculator, OneOutputTest) {
|
||||
}
|
||||
|
||||
// Run the calculator.
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
|
||||
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
|
||||
const std::vector<Packet>& frame_output_packets =
|
||||
runner.Outputs().Index(0).packets;
|
||||
|
||||
EXPECT_EQ(frame_output_packets.size(), 1);
|
||||
}
|
||||
|
||||
// Simple test to make sure that the RealTimeFlowLimiterCalculator waits for all
|
||||
// Simple test to make sure that the FlowLimiterCalculator waits for all
|
||||
// input streams to have at least one packet available before publishing.
|
||||
TEST(RealTimeFlowLimiterCalculator, BasicTest) {
|
||||
TEST(FlowLimiterCalculator, BasicTest) {
|
||||
// Setup the calculator runner and add both ImageFrame and finish packets.
|
||||
CalculatorRunner runner(GetDefaultNode());
|
||||
for (int i = 0; i < kNumImageFrames; ++i) {
|
||||
@@ -117,7 +117,7 @@ TEST(RealTimeFlowLimiterCalculator, BasicTest) {
|
||||
}
|
||||
|
||||
// Run the calculator.
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
|
||||
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
|
||||
const std::vector<Packet>& frame_output_packets =
|
||||
runner.Outputs().Index(0).packets;
|
||||
|
||||
@@ -171,13 +171,11 @@ class CloseCallbackCalculator : public CalculatorBase {
|
||||
};
|
||||
REGISTER_CALCULATOR(CloseCallbackCalculator);
|
||||
|
||||
// Tests demostrating an RealTimeFlowLimiterCalculator operating in a cyclic
|
||||
// graph.
|
||||
// Tests demostrating an FlowLimiterCalculator operating in a cyclic graph.
|
||||
// TODO: clean up these tests.
|
||||
class RealTimeFlowLimiterCalculatorTest : public testing::Test {
|
||||
class FlowLimiterCalculatorTest : public testing::Test {
|
||||
public:
|
||||
RealTimeFlowLimiterCalculatorTest()
|
||||
: enter_semaphore_(0), exit_semaphore_(0) {}
|
||||
FlowLimiterCalculatorTest() : enter_semaphore_(0), exit_semaphore_(0) {}
|
||||
|
||||
void SetUp() override {
|
||||
graph_config_ = InflightGraphConfig();
|
||||
@@ -200,7 +198,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
|
||||
close_count_++;
|
||||
return ::mediapipe::OkStatus();
|
||||
};
|
||||
MEDIAPIPE_ASSERT_OK(graph_.Initialize(
|
||||
MP_ASSERT_OK(graph_.Initialize(
|
||||
graph_config_, {
|
||||
{"max_in_flight", MakePacket<int>(max_in_flight)},
|
||||
{"callback_0", Adopt(new auto(semaphore_0_func))},
|
||||
@@ -211,11 +209,11 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
|
||||
|
||||
// Adds a packet to a graph input stream.
|
||||
void AddPacket(const std::string& input_name, int value) {
|
||||
MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
MP_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
input_name, MakePacket<int>(value).At(Timestamp(value))));
|
||||
}
|
||||
|
||||
// A calculator graph starting with an RealTimeFlowLimiterCalculator and
|
||||
// A calculator graph starting with an FlowLimiterCalculator and
|
||||
// ending with a InFlightFinishCalculator.
|
||||
// Back-edge "finished" limits processing to one frame in-flight.
|
||||
// The two LambdaCalculators are used to keep certain packet sets in flight.
|
||||
@@ -224,7 +222,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
|
||||
input_stream: 'in_1'
|
||||
input_stream: 'in_2'
|
||||
node {
|
||||
calculator: 'RealTimeFlowLimiterCalculator'
|
||||
calculator: 'FlowLimiterCalculator'
|
||||
input_side_packet: 'MAX_IN_FLIGHT:max_in_flight'
|
||||
input_stream: 'in_1'
|
||||
input_stream: 'in_2'
|
||||
@@ -270,19 +268,19 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
|
||||
int close_count_ = 0;
|
||||
};
|
||||
|
||||
// A test demonstrating an RealTimeFlowLimiterCalculator operating in a cyclic
|
||||
// A test demonstrating an FlowLimiterCalculator operating in a cyclic
|
||||
// graph. This test shows that:
|
||||
//
|
||||
// (1) Timestamps are passed through unaltered.
|
||||
// (2) All output streams including the back_edge stream are closed when
|
||||
// the first input stream is closed.
|
||||
//
|
||||
TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) {
|
||||
TEST_F(FlowLimiterCalculatorTest, BackEdgeCloses) {
|
||||
InitializeGraph(1);
|
||||
MEDIAPIPE_ASSERT_OK(graph_.StartRun({}));
|
||||
MP_ASSERT_OK(graph_.StartRun({}));
|
||||
|
||||
auto send_packet = [this](const std::string& input_name, int64 n) {
|
||||
MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
MP_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
input_name, MakePacket<int64>(n).At(Timestamp(n))));
|
||||
};
|
||||
|
||||
@@ -290,14 +288,14 @@ TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) {
|
||||
send_packet("in_1", i * 10);
|
||||
// This next input should be dropped.
|
||||
send_packet("in_1", i * 10 + 5);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
send_packet("in_2", i * 10);
|
||||
exit_semaphore_.Release(1);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
}
|
||||
MEDIAPIPE_EXPECT_OK(graph_.CloseInputStream("in_1"));
|
||||
MEDIAPIPE_EXPECT_OK(graph_.CloseInputStream("in_2"));
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.CloseInputStream("in_1"));
|
||||
MP_EXPECT_OK(graph_.CloseInputStream("in_2"));
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
|
||||
// All output streams are closed and all output packets are delivered,
|
||||
// with stream "in_1" and stream "in_2" closed.
|
||||
@@ -321,19 +319,19 @@ TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) {
|
||||
|
||||
// A test demonstrating that all output streams are closed when all
|
||||
// input streams are closed after the last input packet has been processed.
|
||||
TEST_F(RealTimeFlowLimiterCalculatorTest, AllStreamsClose) {
|
||||
TEST_F(FlowLimiterCalculatorTest, AllStreamsClose) {
|
||||
InitializeGraph(1);
|
||||
MEDIAPIPE_ASSERT_OK(graph_.StartRun({}));
|
||||
MP_ASSERT_OK(graph_.StartRun({}));
|
||||
|
||||
exit_semaphore_.Release(10);
|
||||
for (int i = 0; i < 10; i++) {
|
||||
AddPacket("in_1", i);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
AddPacket("in_2", i);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
}
|
||||
MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
|
||||
EXPECT_EQ(TimestampValues(out_1_packets_), TimestampValues(out_2_packets_));
|
||||
EXPECT_EQ(TimestampValues(out_1_packets_),
|
||||
@@ -341,7 +339,7 @@ TEST_F(RealTimeFlowLimiterCalculatorTest, AllStreamsClose) {
|
||||
EXPECT_EQ(1, close_count_);
|
||||
}
|
||||
|
||||
TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
TEST(FlowLimiterCalculator, TwoStreams) {
|
||||
std::vector<Packet> a_passed;
|
||||
std::vector<Packet> b_passed;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
@@ -351,7 +349,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
input_stream: 'finished'
|
||||
node {
|
||||
name: 'input_dropper'
|
||||
calculator: 'RealTimeFlowLimiterCalculator'
|
||||
calculator: 'FlowLimiterCalculator'
|
||||
input_side_packet: 'MAX_IN_FLIGHT:max_in_flight'
|
||||
input_stream: 'in_a'
|
||||
input_stream: 'in_b'
|
||||
@@ -373,7 +371,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
};
|
||||
|
||||
CalculatorGraph graph_;
|
||||
MEDIAPIPE_EXPECT_OK(graph_.Initialize(
|
||||
MP_EXPECT_OK(graph_.Initialize(
|
||||
graph_config_,
|
||||
{
|
||||
{"max_in_flight", MakePacket<int>(1)},
|
||||
@@ -381,66 +379,66 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
MakePacket<std::function<void(const Packet&)>>(allow_cb)},
|
||||
}));
|
||||
|
||||
MEDIAPIPE_EXPECT_OK(graph_.StartRun({}));
|
||||
MP_EXPECT_OK(graph_.StartRun({}));
|
||||
|
||||
auto send_packet = [&graph_](const std::string& input_name, int n) {
|
||||
MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
MP_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
input_name, MakePacket<int>(n).At(Timestamp(n))));
|
||||
};
|
||||
send_packet("in_a", 1);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(allow, false);
|
||||
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{}));
|
||||
|
||||
send_packet("in_a", 2);
|
||||
send_packet("in_b", 1);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(allow, false);
|
||||
|
||||
send_packet("finished", 1);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(allow, true);
|
||||
|
||||
send_packet("in_b", 2);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(allow, true);
|
||||
|
||||
send_packet("in_b", 3);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
|
||||
EXPECT_EQ(allow, false);
|
||||
|
||||
send_packet("in_b", 4);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
|
||||
EXPECT_EQ(allow, false);
|
||||
|
||||
send_packet("in_a", 3);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
|
||||
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
|
||||
EXPECT_EQ(allow, false);
|
||||
|
||||
send_packet("finished", 3);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
|
||||
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
|
||||
EXPECT_EQ(allow, true);
|
||||
|
||||
MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilDone());
|
||||
MP_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MP_EXPECT_OK(graph_.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST(RealTimeFlowLimiterCalculator, CanConsume) {
|
||||
TEST(FlowLimiterCalculator, CanConsume) {
|
||||
std::vector<Packet> in_sampled_packets_;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
@@ -448,7 +446,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
|
||||
input_stream: 'finished'
|
||||
node {
|
||||
name: 'input_dropper'
|
||||
calculator: 'RealTimeFlowLimiterCalculator'
|
||||
calculator: 'FlowLimiterCalculator'
|
||||
input_side_packet: 'MAX_IN_FLIGHT:max_in_flight'
|
||||
input_stream: 'in'
|
||||
input_stream: 'FINISHED:finished'
|
||||
@@ -467,7 +465,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
|
||||
};
|
||||
|
||||
CalculatorGraph graph_;
|
||||
MEDIAPIPE_EXPECT_OK(graph_.Initialize(
|
||||
MP_EXPECT_OK(graph_.Initialize(
|
||||
graph_config_,
|
||||
{
|
||||
{"max_in_flight", MakePacket<int>(1)},
|
||||
@@ -475,21 +473,21 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
|
||||
MakePacket<std::function<void(const Packet&)>>(allow_cb)},
|
||||
}));
|
||||
|
||||
MEDIAPIPE_EXPECT_OK(graph_.StartRun({}));
|
||||
MP_EXPECT_OK(graph_.StartRun({}));
|
||||
|
||||
auto send_packet = [&graph_](const std::string& input_name, int n) {
|
||||
MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
MP_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
input_name, MakePacket<int>(n).At(Timestamp(n))));
|
||||
};
|
||||
send_packet("in", 1);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(allow, false);
|
||||
EXPECT_EQ(TimestampValues(in_sampled_packets_), (std::vector<int64>{1}));
|
||||
|
||||
MEDIAPIPE_EXPECT_OK(in_sampled_packets_[0].Consume<int>());
|
||||
MP_EXPECT_OK(in_sampled_packets_[0].Consume<int>());
|
||||
|
||||
MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilDone());
|
||||
MP_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MP_EXPECT_OK(graph_.WaitUntilDone());
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
@@ -0,0 +1,163 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/gate_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/header_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
enum GateState {
|
||||
GATE_UNINITIALIZED,
|
||||
GATE_ALLOW,
|
||||
GATE_DISALLOW,
|
||||
};
|
||||
|
||||
std::string ToString(GateState state) {
|
||||
switch (state) {
|
||||
case GATE_UNINITIALIZED:
|
||||
return "UNINITIALIZED";
|
||||
case GATE_ALLOW:
|
||||
return "ALLOW";
|
||||
case GATE_DISALLOW:
|
||||
return "DISALLOW";
|
||||
}
|
||||
DLOG(FATAL) << "Unknown GateState";
|
||||
return "UNKNOWN";
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// Controls whether or not the input packets are passed further along the graph.
|
||||
// Takes multiple data input streams and either an ALLOW or a DISALLOW control
|
||||
// input stream. It outputs an output stream for each input stream that is not
|
||||
// ALLOW or DISALLOW as well as an optional STATE_CHANGE stream which downstream
|
||||
// calculators can use to respond to state-change events.
|
||||
//
|
||||
// If the current ALLOW packet is set to true, the input packets are passed to
|
||||
// their corresponding output stream unchanged. If the ALLOW packet is set to
|
||||
// false, the current input packet is NOT passed to the output stream. If using
|
||||
// DISALLOW, the behavior is opposite of ALLOW.
|
||||
//
|
||||
// By default, an empty packet in the ALLOW or DISALLOW input stream indicates
|
||||
// disallowing the corresponding packets in other input streams. The behavior
|
||||
// can be inverted with a calculator option.
|
||||
//
|
||||
// Intended to be used with the default input stream handler, which synchronizes
|
||||
// all data input streams with the ALLOW/DISALLOW control input stream.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "GateCalculator"
|
||||
// input_stream: "input_stream0"
|
||||
// input_stream: "input_stream1"
|
||||
// input_stream: "input_streamN"
|
||||
// input_stream: "ALLOW:allow" or "DISALLOW:disallow"
|
||||
// output_stream: "STATE_CHANGE:state_change"
|
||||
// output_stream: "output_stream0"
|
||||
// output_stream: "output_stream1"
|
||||
// output_stream: "output_streamN"
|
||||
// }
|
||||
class GateCalculator : public CalculatorBase {
|
||||
public:
|
||||
GateCalculator() {}
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
// Assume that input streams do not have a tag and that gating signal is
|
||||
// tagged either ALLOW or DISALLOW.
|
||||
RET_CHECK(cc->Inputs().HasTag("ALLOW") ^ cc->Inputs().HasTag("DISALLOW"));
|
||||
const int num_data_streams = cc->Inputs().NumEntries("");
|
||||
RET_CHECK_GE(num_data_streams, 1);
|
||||
RET_CHECK_EQ(cc->Outputs().NumEntries(""), num_data_streams)
|
||||
<< "Number of data output streams must match with data input streams.";
|
||||
|
||||
for (int i = 0; i < num_data_streams; ++i) {
|
||||
cc->Inputs().Get("", i).SetAny();
|
||||
cc->Outputs().Get("", i).SetSameAs(&cc->Inputs().Get("", i));
|
||||
}
|
||||
if (cc->Inputs().HasTag("ALLOW")) {
|
||||
cc->Inputs().Tag("ALLOW").Set<bool>();
|
||||
} else {
|
||||
cc->Inputs().Tag("DISALLOW").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag("STATE_CHANGE")) {
|
||||
cc->Outputs().Tag("STATE_CHANGE").Set<bool>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) final {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
num_data_streams_ = cc->Inputs().NumEntries("");
|
||||
last_gate_state_ = GATE_UNINITIALIZED;
|
||||
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
|
||||
|
||||
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
|
||||
empty_packets_as_allow_ = options.empty_packets_as_allow();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) final {
|
||||
bool allow = empty_packets_as_allow_;
|
||||
if (cc->Inputs().HasTag("ALLOW") && !cc->Inputs().Tag("ALLOW").IsEmpty()) {
|
||||
allow = cc->Inputs().Tag("ALLOW").Get<bool>();
|
||||
}
|
||||
if (cc->Inputs().HasTag("DISALLOW") &&
|
||||
!cc->Inputs().Tag("DISALLOW").IsEmpty()) {
|
||||
allow = !cc->Inputs().Tag("DISALLOW").Get<bool>();
|
||||
}
|
||||
|
||||
const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
|
||||
|
||||
if (cc->Outputs().HasTag("STATE_CHANGE")) {
|
||||
if (last_gate_state_ != GATE_UNINITIALIZED &&
|
||||
last_gate_state_ != new_gate_state) {
|
||||
VLOG(2) << "State transition in " << cc->NodeName() << " @ "
|
||||
<< cc->InputTimestamp().Value() << " from "
|
||||
<< ToString(last_gate_state_) << " to "
|
||||
<< ToString(new_gate_state);
|
||||
cc->Outputs()
|
||||
.Tag("STATE_CHANGE")
|
||||
.AddPacket(MakePacket<bool>(allow).At(cc->InputTimestamp()));
|
||||
}
|
||||
}
|
||||
last_gate_state_ = new_gate_state;
|
||||
|
||||
if (!allow) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
// Process data streams.
|
||||
for (int i = 0; i < num_data_streams_; ++i) {
|
||||
if (!cc->Inputs().Get("", i).IsEmpty()) {
|
||||
cc->Outputs().Get("", i).AddPacket(cc->Inputs().Get("", i).Value());
|
||||
}
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
GateState last_gate_state_ = GATE_UNINITIALIZED;
|
||||
int num_data_streams_;
|
||||
bool empty_packets_as_allow_;
|
||||
};
|
||||
REGISTER_CALCULATOR(GateCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message GateCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional GateCalculatorOptions ext = 261754847;
|
||||
}
|
||||
|
||||
// By default an empty packet in the ALLOW or DISALLOW input stream indicates
|
||||
// disallowing the corresponding packets in the data input streams. Setting
|
||||
// this option to true inverts that, allowing the data packets to go through.
|
||||
optional bool empty_packets_as_allow = 1;
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
class GateCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void RunTimeStep(int64 timestamp, const std::string& control_tag,
|
||||
bool control) {
|
||||
runner_->MutableInputs()->Get("", 0).packets.push_back(
|
||||
MakePacket<bool>(true).At(Timestamp(timestamp)));
|
||||
runner_->MutableInputs()
|
||||
->Tag(control_tag)
|
||||
.packets.push_back(MakePacket<bool>(control).At(Timestamp(timestamp)));
|
||||
|
||||
MP_ASSERT_OK(runner_->Run()) << "Calculator execution failed.";
|
||||
}
|
||||
|
||||
void SetRunner(const std::string& proto) {
|
||||
runner_ = absl::make_unique<CalculatorRunner>(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(proto));
|
||||
}
|
||||
|
||||
CalculatorRunner* runner() { return runner_.get(); }
|
||||
|
||||
private:
|
||||
std::unique_ptr<CalculatorRunner> runner_;
|
||||
};
|
||||
|
||||
TEST_F(GateCalculatorTest, Allow) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
input_stream: "ALLOW:gating_stream"
|
||||
output_stream: "test_output"
|
||||
)");
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, "ALLOW", true);
|
||||
constexpr int64 kTimestampValue1 = 43;
|
||||
RunTimeStep(kTimestampValue1, "ALLOW", false);
|
||||
constexpr int64 kTimestampValue2 = 44;
|
||||
RunTimeStep(kTimestampValue2, "ALLOW", true);
|
||||
constexpr int64 kTimestampValue3 = 45;
|
||||
RunTimeStep(kTimestampValue3, "ALLOW", false);
|
||||
|
||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||
ASSERT_EQ(2, output.size());
|
||||
EXPECT_EQ(kTimestampValue0, output[0].Timestamp().Value());
|
||||
EXPECT_EQ(kTimestampValue2, output[1].Timestamp().Value());
|
||||
EXPECT_EQ(true, output[0].Get<bool>());
|
||||
EXPECT_EQ(true, output[1].Get<bool>());
|
||||
}
|
||||
|
||||
TEST_F(GateCalculatorTest, Disallow) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
input_stream: "DISALLOW:gating_stream"
|
||||
output_stream: "test_output"
|
||||
)");
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, "DISALLOW", true);
|
||||
constexpr int64 kTimestampValue1 = 43;
|
||||
RunTimeStep(kTimestampValue1, "DISALLOW", false);
|
||||
constexpr int64 kTimestampValue2 = 44;
|
||||
RunTimeStep(kTimestampValue2, "DISALLOW", true);
|
||||
constexpr int64 kTimestampValue3 = 45;
|
||||
RunTimeStep(kTimestampValue3, "DISALLOW", false);
|
||||
|
||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||
ASSERT_EQ(2, output.size());
|
||||
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
|
||||
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
|
||||
EXPECT_EQ(true, output[0].Get<bool>());
|
||||
EXPECT_EQ(true, output[1].Get<bool>());
|
||||
}
|
||||
|
||||
TEST_F(GateCalculatorTest, AllowWithStateChange) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
input_stream: "ALLOW:gating_stream"
|
||||
output_stream: "test_output"
|
||||
output_stream: "STATE_CHANGE:state_changed"
|
||||
)");
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, "ALLOW", false);
|
||||
constexpr int64 kTimestampValue1 = 43;
|
||||
RunTimeStep(kTimestampValue1, "ALLOW", true);
|
||||
constexpr int64 kTimestampValue2 = 44;
|
||||
RunTimeStep(kTimestampValue2, "ALLOW", true);
|
||||
constexpr int64 kTimestampValue3 = 45;
|
||||
RunTimeStep(kTimestampValue3, "ALLOW", false);
|
||||
|
||||
const std::vector<Packet>& output =
|
||||
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
||||
ASSERT_EQ(2, output.size());
|
||||
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
|
||||
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
|
||||
EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
|
||||
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
|
||||
}
|
||||
|
||||
TEST_F(GateCalculatorTest, DisallowWithStateChange) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
input_stream: "DISALLOW:gating_stream"
|
||||
output_stream: "test_output"
|
||||
output_stream: "STATE_CHANGE:state_changed"
|
||||
)");
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, "DISALLOW", true);
|
||||
constexpr int64 kTimestampValue1 = 43;
|
||||
RunTimeStep(kTimestampValue1, "DISALLOW", false);
|
||||
constexpr int64 kTimestampValue2 = 44;
|
||||
RunTimeStep(kTimestampValue2, "DISALLOW", false);
|
||||
constexpr int64 kTimestampValue3 = 45;
|
||||
RunTimeStep(kTimestampValue3, "DISALLOW", true);
|
||||
|
||||
const std::vector<Packet>& output =
|
||||
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
||||
ASSERT_EQ(2, output.size());
|
||||
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
|
||||
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
|
||||
EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
|
||||
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
|
||||
}
|
||||
|
||||
// Must not detect disallow value for first timestamp as a state change.
|
||||
TEST_F(GateCalculatorTest, DisallowInitialNoStateTransition) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
input_stream: "DISALLOW:gating_stream"
|
||||
output_stream: "test_output"
|
||||
output_stream: "STATE_CHANGE:state_changed"
|
||||
)");
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, "DISALLOW", false);
|
||||
|
||||
const std::vector<Packet>& output =
|
||||
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
||||
ASSERT_EQ(0, output.size());
|
||||
}
|
||||
|
||||
// Must not detect allow value for first timestamp as a state change.
|
||||
TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
input_stream: "ALLOW:gating_stream"
|
||||
output_stream: "test_output"
|
||||
output_stream: "STATE_CHANGE:state_changed"
|
||||
)");
|
||||
|
||||
constexpr int64 kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, "ALLOW", true);
|
||||
|
||||
const std::vector<Packet>& output =
|
||||
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
||||
ASSERT_EQ(0, output.size());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -146,7 +146,7 @@ class ImmediateMuxCalculatorTest : public ::testing::Test {
|
||||
ASSERT_TRUE(proto_ns::TextFormat::ParseFromString(R"(
|
||||
input_stream: "input_packets_0"
|
||||
node {
|
||||
calculator: 'RealTimeFlowLimiterCalculator'
|
||||
calculator: 'FlowLimiterCalculator'
|
||||
input_stream_handler {
|
||||
input_stream_handler: 'ImmediateInputStreamHandler'
|
||||
}
|
||||
@@ -217,23 +217,23 @@ class ImmediateMuxCalculatorTest : public ::testing::Test {
|
||||
|
||||
// Start running the graph.
|
||||
CalculatorGraph graph;
|
||||
MEDIAPIPE_ASSERT_OK(graph.Initialize(graph_config_));
|
||||
MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config_));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
// Send each packet to the graph in the specified order.
|
||||
for (int t = 0; t < input_sets.size(); t++) {
|
||||
const std::vector<Packet>& input_set = input_sets[t];
|
||||
MEDIAPIPE_EXPECT_OK(graph.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph.WaitUntilIdle());
|
||||
for (int i = 0; i < input_set.size(); i++) {
|
||||
const Packet& packet = input_set[i];
|
||||
if (!IsNone(packet)) {
|
||||
MEDIAPIPE_EXPECT_OK(graph.AddPacketToInputStream(
|
||||
MP_EXPECT_OK(graph.AddPacketToInputStream(
|
||||
absl::StrCat("input_packets_", i), packet));
|
||||
}
|
||||
}
|
||||
}
|
||||
MEDIAPIPE_ASSERT_OK(graph.CloseAllInputStreams());
|
||||
MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
|
||||
MP_ASSERT_OK(graph.CloseAllInputStreams());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
CalculatorGraphConfig graph_config_;
|
||||
@@ -335,22 +335,22 @@ TEST_F(ImmediateMuxCalculatorTest, Demux) {
|
||||
|
||||
// Start the graph and add five input packets.
|
||||
CalculatorGraph graph;
|
||||
MEDIAPIPE_ASSERT_OK(graph.Initialize(
|
||||
graph_config_, {
|
||||
{"callback_0", Adopt(new auto(wait_0))},
|
||||
{"callback_1", Adopt(new auto(wait_1))},
|
||||
}));
|
||||
MEDIAPIPE_ASSERT_OK(graph.ObserveOutputStream("output_packets_0", out_cb));
|
||||
MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
|
||||
MEDIAPIPE_EXPECT_OK(
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config_,
|
||||
{
|
||||
{"callback_0", Adopt(new auto(wait_0))},
|
||||
{"callback_1", Adopt(new auto(wait_1))},
|
||||
}));
|
||||
MP_ASSERT_OK(graph.ObserveOutputStream("output_packets_0", out_cb));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_EXPECT_OK(
|
||||
graph.AddPacketToInputStream("input_packets_0", PacketAt(10000)));
|
||||
MEDIAPIPE_EXPECT_OK(
|
||||
MP_EXPECT_OK(
|
||||
graph.AddPacketToInputStream("input_packets_0", PacketAt(20000)));
|
||||
MEDIAPIPE_EXPECT_OK(
|
||||
MP_EXPECT_OK(
|
||||
graph.AddPacketToInputStream("input_packets_0", PacketAt(30000)));
|
||||
MEDIAPIPE_EXPECT_OK(
|
||||
MP_EXPECT_OK(
|
||||
graph.AddPacketToInputStream("input_packets_0", PacketAt(40000)));
|
||||
MEDIAPIPE_EXPECT_OK(
|
||||
MP_EXPECT_OK(
|
||||
graph.AddPacketToInputStream("input_packets_0", PacketAt(50000)));
|
||||
|
||||
// Release the outputs in order 20000, 10000, 30000, 50000, 40000.
|
||||
@@ -362,8 +362,8 @@ TEST_F(ImmediateMuxCalculatorTest, Demux) {
|
||||
semaphore_0.Release(1); // 50000
|
||||
wait_for([&] { return out_packets.size() >= 3; });
|
||||
semaphore_1.Release(1); // 40000
|
||||
MEDIAPIPE_ASSERT_OK(graph.CloseAllInputStreams());
|
||||
MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
|
||||
MP_ASSERT_OK(graph.CloseAllInputStreams());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
|
||||
// Output packets 10000 and 40000 are superseded and dropped.
|
||||
EXPECT_THAT(TimestampValues(out_packets), ElementsAre(20000, 30000, 50000));
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
// Perform a (left) matrix multiply. Meaning (output = A * input)
|
||||
// where A is the matrix which is provided as an input side packet.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "MatrixMultiplyCalculator"
|
||||
// input_stream: "samples"
|
||||
// output_stream: "multiplied_samples"
|
||||
// input_side_packet: "multiplication_matrix"
|
||||
// }
|
||||
class MatrixMultiplyCalculator : public CalculatorBase {
|
||||
public:
|
||||
MatrixMultiplyCalculator() {}
|
||||
~MatrixMultiplyCalculator() override {}
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(MatrixMultiplyCalculator);
|
||||
|
||||
// static
|
||||
::mediapipe::Status MatrixMultiplyCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<Matrix>();
|
||||
cc->Outputs().Index(0).Set<Matrix>();
|
||||
cc->InputSidePackets().Index(0).Set<Matrix>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status MatrixMultiplyCalculator::Open(CalculatorContext* cc) {
|
||||
// The output is at the same timestamp as the input.
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status MatrixMultiplyCalculator::Process(CalculatorContext* cc) {
|
||||
Matrix* multiplied = new Matrix();
|
||||
*multiplied = cc->InputSidePackets().Index(0).Get<Matrix>() *
|
||||
cc->Inputs().Index(0).Get<Matrix>();
|
||||
cc->Outputs().Index(0).Add(multiplied, cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,239 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
// A 3x4 Matrix of random integers in [0,1000).
|
||||
const char kMatrixText[] =
|
||||
"rows: 3\n"
|
||||
"cols: 4\n"
|
||||
"packed_data: 387\n"
|
||||
"packed_data: 940\n"
|
||||
"packed_data: 815\n"
|
||||
"packed_data: 825\n"
|
||||
"packed_data: 997\n"
|
||||
"packed_data: 884\n"
|
||||
"packed_data: 419\n"
|
||||
"packed_data: 763\n"
|
||||
"packed_data: 123\n"
|
||||
"packed_data: 30\n"
|
||||
"packed_data: 825\n"
|
||||
"packed_data: 299\n";
|
||||
|
||||
// A 4x20 Matrix of random integers in [0,10).
|
||||
// Each column of this matrix is a sample.
|
||||
const char kSamplesText[] =
|
||||
"rows: 4\n"
|
||||
"cols: 20\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 9\n"
|
||||
"packed_data: 5\n"
|
||||
"packed_data: 9\n"
|
||||
"packed_data: 6\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 0\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 1\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 2\n"
|
||||
"packed_data: 4\n"
|
||||
"packed_data: 5\n"
|
||||
"packed_data: 0\n"
|
||||
"packed_data: 4\n"
|
||||
"packed_data: 6\n"
|
||||
"packed_data: 0\n"
|
||||
"packed_data: 1\n"
|
||||
"packed_data: 2\n"
|
||||
"packed_data: 0\n"
|
||||
"packed_data: 2\n"
|
||||
"packed_data: 0\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 1\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 4\n"
|
||||
"packed_data: 9\n"
|
||||
"packed_data: 8\n"
|
||||
"packed_data: 8\n"
|
||||
"packed_data: 6\n"
|
||||
"packed_data: 4\n"
|
||||
"packed_data: 6\n"
|
||||
"packed_data: 8\n"
|
||||
"packed_data: 1\n"
|
||||
"packed_data: 9\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 5\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 5\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 5\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 6\n"
|
||||
"packed_data: 4\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 2\n"
|
||||
"packed_data: 5\n"
|
||||
"packed_data: 4\n"
|
||||
"packed_data: 8\n"
|
||||
"packed_data: 1\n"
|
||||
"packed_data: 0\n"
|
||||
"packed_data: 2\n"
|
||||
"packed_data: 0\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 4\n"
|
||||
"packed_data: 6\n"
|
||||
"packed_data: 6\n"
|
||||
"packed_data: 8\n"
|
||||
"packed_data: 5\n"
|
||||
"packed_data: 5\n"
|
||||
"packed_data: 8\n"
|
||||
"packed_data: 9\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 3\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 2\n"
|
||||
"packed_data: 7\n"
|
||||
"packed_data: 8\n"
|
||||
"packed_data: 2\n"
|
||||
"packed_data: 1\n"
|
||||
"packed_data: 1\n"
|
||||
"packed_data: 4\n"
|
||||
"packed_data: 1\n"
|
||||
"packed_data: 1\n"
|
||||
"packed_data: 7\n";
|
||||
|
||||
// A 3x20 Matrix of expected values for the result of the matrix multiply
|
||||
// computed using R.
|
||||
// Each column of this matrix is an expected output.
|
||||
const char kExpectedText[] =
|
||||
"rows: 3\n"
|
||||
"cols: 20\n"
|
||||
"packed_data: 12499\n"
|
||||
"packed_data: 26793\n"
|
||||
"packed_data: 16967\n"
|
||||
"packed_data: 5007\n"
|
||||
"packed_data: 14406\n"
|
||||
"packed_data: 9635\n"
|
||||
"packed_data: 4179\n"
|
||||
"packed_data: 7870\n"
|
||||
"packed_data: 4434\n"
|
||||
"packed_data: 5793\n"
|
||||
"packed_data: 12045\n"
|
||||
"packed_data: 8876\n"
|
||||
"packed_data: 2801\n"
|
||||
"packed_data: 8053\n"
|
||||
"packed_data: 5611\n"
|
||||
"packed_data: 1740\n"
|
||||
"packed_data: 4469\n"
|
||||
"packed_data: 2665\n"
|
||||
"packed_data: 8108\n"
|
||||
"packed_data: 18396\n"
|
||||
"packed_data: 10186\n"
|
||||
"packed_data: 12330\n"
|
||||
"packed_data: 23374\n"
|
||||
"packed_data: 15526\n"
|
||||
"packed_data: 9611\n"
|
||||
"packed_data: 21804\n"
|
||||
"packed_data: 14776\n"
|
||||
"packed_data: 8241\n"
|
||||
"packed_data: 17979\n"
|
||||
"packed_data: 11989\n"
|
||||
"packed_data: 8429\n"
|
||||
"packed_data: 18921\n"
|
||||
"packed_data: 9819\n"
|
||||
"packed_data: 6270\n"
|
||||
"packed_data: 13689\n"
|
||||
"packed_data: 7031\n"
|
||||
"packed_data: 9472\n"
|
||||
"packed_data: 19210\n"
|
||||
"packed_data: 13634\n"
|
||||
"packed_data: 8567\n"
|
||||
"packed_data: 12499\n"
|
||||
"packed_data: 10455\n"
|
||||
"packed_data: 2151\n"
|
||||
"packed_data: 7469\n"
|
||||
"packed_data: 3195\n"
|
||||
"packed_data: 10774\n"
|
||||
"packed_data: 21851\n"
|
||||
"packed_data: 12673\n"
|
||||
"packed_data: 12516\n"
|
||||
"packed_data: 25318\n"
|
||||
"packed_data: 14347\n"
|
||||
"packed_data: 7984\n"
|
||||
"packed_data: 17100\n"
|
||||
"packed_data: 10972\n"
|
||||
"packed_data: 5195\n"
|
||||
"packed_data: 11102\n"
|
||||
"packed_data: 8710\n"
|
||||
"packed_data: 3002\n"
|
||||
"packed_data: 11295\n"
|
||||
"packed_data: 6360\n";
|
||||
|
||||
// Send a number of samples through the MatrixMultiplyCalculator.
|
||||
TEST(MatrixMultiplyCalculatorTest, Multiply) {
|
||||
CalculatorRunner runner("MatrixMultiplyCalculator", "", 1, 1, 1);
|
||||
Matrix* matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText, matrix);
|
||||
runner.MutableSidePackets()->Index(0) = Adopt(matrix);
|
||||
|
||||
Matrix samples;
|
||||
MatrixFromTextProto(kSamplesText, &samples);
|
||||
Matrix expected;
|
||||
MatrixFromTextProto(kExpectedText, &expected);
|
||||
CHECK_EQ(samples.cols(), expected.cols());
|
||||
|
||||
for (int i = 0; i < samples.cols(); ++i) {
|
||||
// Take a column from samples and produce a packet with just that
|
||||
// column in it as an input sample for the calculator.
|
||||
Eigen::MatrixXf* sample = new Eigen::MatrixXf(samples.block(0, i, 4, 1));
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(sample).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
EXPECT_EQ(runner.MutableInputs()->Index(0).packets.size(),
|
||||
runner.Outputs().Index(0).packets.size());
|
||||
|
||||
int i = 0;
|
||||
for (const Packet& output : runner.Outputs().Index(0).packets) {
|
||||
EXPECT_EQ(Timestamp(i), output.Timestamp());
|
||||
const Eigen::MatrixXf& result = output.Get<Matrix>();
|
||||
ASSERT_EQ(3, result.rows());
|
||||
EXPECT_NEAR((expected.block(0, i, 3, 1) - result).cwiseAbs().sum(), 0.0,
|
||||
1e-5);
|
||||
++i;
|
||||
}
|
||||
EXPECT_EQ(samples.cols(), i);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,123 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Subtract input matrix from the side input matrix and vice versa. The matrices
|
||||
// must have the same dimension.
|
||||
// Based on the tag (MINUEND vs SUBTRAHEND), the matrices in the input stream
|
||||
// and input side packet can be either subtrahend or minuend. The output matrix
|
||||
// is generated by performing minuend matrix - subtrahend matrix.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "MatrixSubtractCalculator"
|
||||
// input_stream: "MINUEND:input_matrix"
|
||||
// input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
// output_stream: "output_matrix"
|
||||
// }
|
||||
//
|
||||
// or
|
||||
//
|
||||
// node {
|
||||
// calculator: "MatrixSubtractCalculator"
|
||||
// input_stream: "SUBTRAHEND:input_matrix"
|
||||
// input_side_packet: "MINUEND:side_matrix"
|
||||
// output_stream: "output_matrix"
|
||||
// }
|
||||
class MatrixSubtractCalculator : public CalculatorBase {
|
||||
public:
|
||||
MatrixSubtractCalculator() {}
|
||||
~MatrixSubtractCalculator() override {}
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
bool subtract_from_input_ = false;
|
||||
};
|
||||
REGISTER_CALCULATOR(MatrixSubtractCalculator);
|
||||
|
||||
// static
|
||||
::mediapipe::Status MatrixSubtractCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
if (cc->Inputs().NumEntries() != 1 ||
|
||||
cc->InputSidePackets().NumEntries() != 1) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"MatrixSubtractCalculator only accepts exactly one input stream and "
|
||||
"one "
|
||||
"input side packet");
|
||||
}
|
||||
if (cc->Inputs().HasTag("MINUEND") &&
|
||||
cc->InputSidePackets().HasTag("SUBTRAHEND")) {
|
||||
cc->Inputs().Tag("MINUEND").Set<Matrix>();
|
||||
cc->InputSidePackets().Tag("SUBTRAHEND").Set<Matrix>();
|
||||
} else if (cc->Inputs().HasTag("SUBTRAHEND") &&
|
||||
cc->InputSidePackets().HasTag("MINUEND")) {
|
||||
cc->Inputs().Tag("SUBTRAHEND").Set<Matrix>();
|
||||
cc->InputSidePackets().Tag("MINUEND").Set<Matrix>();
|
||||
} else {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"Must specify exactly one minuend and one subtrahend.");
|
||||
}
|
||||
cc->Outputs().Index(0).Set<Matrix>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status MatrixSubtractCalculator::Open(CalculatorContext* cc) {
|
||||
// The output is at the same timestamp as the input.
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
if (cc->Inputs().HasTag("MINUEND")) {
|
||||
subtract_from_input_ = true;
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status MatrixSubtractCalculator::Process(CalculatorContext* cc) {
|
||||
Matrix* subtracted = new Matrix();
|
||||
if (subtract_from_input_) {
|
||||
const Matrix& input_matrix = cc->Inputs().Tag("MINUEND").Get<Matrix>();
|
||||
const Matrix& side_input_matrix =
|
||||
cc->InputSidePackets().Tag("SUBTRAHEND").Get<Matrix>();
|
||||
if (input_matrix.rows() != side_input_matrix.rows() ||
|
||||
input_matrix.cols() != side_input_matrix.cols()) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"Input matrix and the input side matrix must have the same "
|
||||
"dimension.");
|
||||
}
|
||||
*subtracted = input_matrix - side_input_matrix;
|
||||
} else {
|
||||
const Matrix& input_matrix = cc->Inputs().Tag("SUBTRAHEND").Get<Matrix>();
|
||||
const Matrix& side_input_matrix =
|
||||
cc->InputSidePackets().Tag("MINUEND").Get<Matrix>();
|
||||
if (input_matrix.rows() != side_input_matrix.rows() ||
|
||||
input_matrix.cols() != side_input_matrix.cols()) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"Input matrix and the input side matrix must have the same "
|
||||
"dimension.");
|
||||
}
|
||||
*subtracted = side_input_matrix - input_matrix;
|
||||
}
|
||||
cc->Outputs().Index(0).Add(subtracted, cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,157 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
// A 3x4 Matrix of random integers in [0,1000).
|
||||
const char kMatrixText[] =
|
||||
"rows: 3\n"
|
||||
"cols: 4\n"
|
||||
"packed_data: 387\n"
|
||||
"packed_data: 940\n"
|
||||
"packed_data: 815\n"
|
||||
"packed_data: 825\n"
|
||||
"packed_data: 997\n"
|
||||
"packed_data: 884\n"
|
||||
"packed_data: 419\n"
|
||||
"packed_data: 763\n"
|
||||
"packed_data: 123\n"
|
||||
"packed_data: 30\n"
|
||||
"packed_data: 825\n"
|
||||
"packed_data: 299\n";
|
||||
|
||||
const char kMatrixText2[] =
|
||||
"rows: 3\n"
|
||||
"cols: 4\n"
|
||||
"packed_data: 388\n"
|
||||
"packed_data: 941\n"
|
||||
"packed_data: 816\n"
|
||||
"packed_data: 826\n"
|
||||
"packed_data: 998\n"
|
||||
"packed_data: 885\n"
|
||||
"packed_data: 420\n"
|
||||
"packed_data: 764\n"
|
||||
"packed_data: 124\n"
|
||||
"packed_data: 31\n"
|
||||
"packed_data: 826\n"
|
||||
"packed_data: 300\n";
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "input_matrix"
|
||||
input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
input_side_packet: "MINUEND:side_matrix2"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
auto status = runner.Run();
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
testing::HasSubstr(
|
||||
"only accepts exactly one input stream and one input side packet"));
|
||||
}
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
input_stream: "SUBTRAHEND:side_matrix2"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
auto status = runner.Run();
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
testing::HasSubstr("specify exactly one minuend and one subtrahend."));
|
||||
}
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "MINUEND:input_matrix"
|
||||
input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
Matrix* side_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText, side_matrix);
|
||||
runner.MutableSidePackets()->Tag("SUBTRAHEND") = Adopt(side_matrix);
|
||||
|
||||
Matrix* input_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText2, input_matrix);
|
||||
runner.MutableInputs()->Tag("MINUEND").packets.push_back(
|
||||
Adopt(input_matrix).At(Timestamp(0)));
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
EXPECT_EQ(1, runner.Outputs().Index(0).packets.size());
|
||||
|
||||
EXPECT_EQ(Timestamp(0), runner.Outputs().Index(0).packets[0].Timestamp());
|
||||
const Eigen::MatrixXf& result =
|
||||
runner.Outputs().Index(0).packets[0].Get<Matrix>();
|
||||
ASSERT_EQ(3, result.rows());
|
||||
ASSERT_EQ(4, result.cols());
|
||||
EXPECT_NEAR(result.sum(), 12, 1e-5);
|
||||
}
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "SUBTRAHEND:input_matrix"
|
||||
input_side_packet: "MINUEND:side_matrix"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
Matrix* side_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText, side_matrix);
|
||||
runner.MutableSidePackets()->Tag("MINUEND") = Adopt(side_matrix);
|
||||
|
||||
Matrix* input_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText2, input_matrix);
|
||||
runner.MutableInputs()
|
||||
->Tag("SUBTRAHEND")
|
||||
.packets.push_back(Adopt(input_matrix).At(Timestamp(0)));
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
EXPECT_EQ(1, runner.Outputs().Index(0).packets.size());
|
||||
|
||||
EXPECT_EQ(Timestamp(0), runner.Outputs().Index(0).packets[0].Timestamp());
|
||||
const Eigen::MatrixXf& result =
|
||||
runner.Outputs().Index(0).packets[0].Get<Matrix>();
|
||||
ASSERT_EQ(3, result.rows());
|
||||
ASSERT_EQ(4, result.cols());
|
||||
EXPECT_NEAR(result.sum(), -12, 1e-5);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Defines MatrixToVectorCalculator.
|
||||
#include <math.h>
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/tool/status_util.h"
|
||||
#include "mediapipe/util/time_series_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// A calculator that converts a Matrix M to a vector containing all the
|
||||
// entries of M in column-major order.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "MatrixToVectorCalculator"
|
||||
// input_stream: "input_matrix"
|
||||
// output_stream: "column_major_vector"
|
||||
// }
|
||||
class MatrixToVectorCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<Matrix>(
|
||||
// Input Packet containing a Matrix.
|
||||
);
|
||||
cc->Outputs().Index(0).Set<std::vector<float>>(
|
||||
// Output Packet containing a vector, one for each input Packet.
|
||||
);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
|
||||
// Outputs a packet containing a vector for each input packet.
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(MatrixToVectorCalculator);
|
||||
|
||||
::mediapipe::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) {
|
||||
// Inform the framework that we don't alter timestamps.
|
||||
cc->SetOffset(mediapipe::TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
|
||||
const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
|
||||
auto output = absl::make_unique<std::vector<float>>();
|
||||
|
||||
// The following lines work to convert the Matrix to a vector because Matrix
|
||||
// is an Eigen::MatrixXf and Eigen uses column-major layout by default.
|
||||
output->resize(input.rows() * input.cols());
|
||||
auto output_as_matrix =
|
||||
Eigen::Map<Matrix>(output->data(), input.rows(), input.cols());
|
||||
output_as_matrix = input;
|
||||
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,88 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
#include "mediapipe/util/time_series_test_util.h"
|
||||
#include "mediapipe/util/time_series_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
class MatrixToVectorCalculatorTest
|
||||
: public mediapipe::TimeSeriesCalculatorTest<mediapipe::NoOptions> {
|
||||
protected:
|
||||
void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
|
||||
|
||||
void AppendInput(const std::vector<float>& column_major_data,
|
||||
int64 timestamp) {
|
||||
ASSERT_EQ(num_input_samples_ * num_input_channels_,
|
||||
column_major_data.size());
|
||||
Eigen::Map<const Matrix> data_map(&column_major_data[0],
|
||||
num_input_channels_, num_input_samples_);
|
||||
AppendInputPacket(new Matrix(data_map), timestamp);
|
||||
}
|
||||
|
||||
void SetInputStreamParameters(int num_channels, int num_samples) {
|
||||
num_input_channels_ = num_channels;
|
||||
num_input_samples_ = num_samples;
|
||||
input_sample_rate_ = 100;
|
||||
input_packet_rate_ = 20.0;
|
||||
}
|
||||
|
||||
void SetInputHeader(int num_channels, int num_samples) {
|
||||
SetInputStreamParameters(num_channels, num_samples);
|
||||
FillInputHeader();
|
||||
}
|
||||
|
||||
void CheckOutputPacket(int packet, std::vector<float> expected_vector) {
|
||||
const auto& actual_vector =
|
||||
runner_->Outputs().Index(0).packets[packet].Get<std::vector<float>>();
|
||||
EXPECT_THAT(actual_vector, testing::ContainerEq(expected_vector));
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(MatrixToVectorCalculatorTest, SingleRow) {
|
||||
InitializeGraph();
|
||||
SetInputHeader(1, 4); // 1 channel x 4 samples
|
||||
const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0};
|
||||
AppendInput(data_vector, 0);
|
||||
MP_ASSERT_OK(RunGraph());
|
||||
CheckOutputPacket(0, data_vector);
|
||||
}
|
||||
|
||||
TEST_F(MatrixToVectorCalculatorTest, RegularMatrix) {
|
||||
InitializeGraph();
|
||||
SetInputHeader(4, 2); // 4 channels x 2 samples
|
||||
// Actual data matrix is the transpose of the appearance below.
|
||||
const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0,
|
||||
5.0, 6.0, 7.0, 8.0};
|
||||
AppendInput(data_vector, 0);
|
||||
|
||||
MP_ASSERT_OK(RunGraph());
|
||||
CheckOutputPacket(0, data_vector);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,91 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// This calculator takes a set of input streams and combines them into a single
|
||||
// output stream. The packets from different streams do not need to contain the
|
||||
// same type. If there are packets arriving at the same time from two or more
|
||||
// input streams, the packet corresponding to the input stream with the smallest
|
||||
// index is passed to the output and the rest are ignored.
|
||||
//
|
||||
// Example use-case:
|
||||
// Suppose we have two (or more) different algorithms for detecting shot
|
||||
// boundaries and we need to merge their packets into a single stream. The
|
||||
// algorithms may emit shot boundaries at the same time and their output types
|
||||
// may not be compatible. Subsequent calculators that process the merged stream
|
||||
// may be interested only in the timestamps of the shot boundary packets and so
|
||||
// it may not even need to inspect the values stored inside the packets.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "MergeCalculator"
|
||||
// input_stream: "shot_info1"
|
||||
// input_stream: "shot_info2"
|
||||
// input_stream: "shot_info3"
|
||||
// output_stream: "merged_shot_infos"
|
||||
// }
|
||||
//
|
||||
class MergeCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK_GT(cc->Inputs().NumEntries(), 0)
|
||||
<< "Needs at least one input stream";
|
||||
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
|
||||
if (cc->Inputs().NumEntries() == 1) {
|
||||
LOG(WARNING)
|
||||
<< "MergeCalculator expects multiple input streams to merge but is "
|
||||
"receiving only one. Make sure the calculator is configured "
|
||||
"correctly or consider removing this calculator to reduce "
|
||||
"unnecessary overhead.";
|
||||
}
|
||||
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
cc->Inputs().Index(i).SetAny();
|
||||
}
|
||||
cc->Outputs().Index(0).SetAny();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) final {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) final {
|
||||
// Output the packet from the first input stream with a packet ready at this
|
||||
// timestamp.
|
||||
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
|
||||
if (!cc->Inputs().Index(i).IsEmpty()) {
|
||||
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(i).Value());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
}
|
||||
|
||||
LOG(WARNING) << "Empty input packets at timestamp "
|
||||
<< cc->InputTimestamp().Value();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(MergeCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,139 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
// Checks that the calculator fails if no input streams are provided.
|
||||
TEST(InvariantMergeInputStreamsCalculator, NoInputStreamsMustFail) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MergeCalculator"
|
||||
output_stream: "merged_output"
|
||||
)"));
|
||||
// Expect calculator to fail.
|
||||
ASSERT_FALSE(runner.Run().ok());
|
||||
}
|
||||
|
||||
// Checks that the calculator fails with an incorrect number of output streams.
|
||||
TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
|
||||
CalculatorRunner runner1(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
)"));
|
||||
// Expect calculator to fail.
|
||||
EXPECT_FALSE(runner1.Run().ok());
|
||||
|
||||
CalculatorRunner runner2(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
output_stream: "output1"
|
||||
output_stream: "output2"
|
||||
)"));
|
||||
// Expect calculator to fail.
|
||||
ASSERT_FALSE(runner2.Run().ok());
|
||||
}
|
||||
|
||||
// Ensures two streams with differing types can be merged correctly.
|
||||
TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
|
||||
TestMergingTwoStreams) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
output_stream: "combined_output"
|
||||
)"));
|
||||
|
||||
// input1: integers 10, 20, 30, occurring at times 10, 20, 30.
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(new int(10)).At(Timestamp(10)));
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(new int(20)).At(Timestamp(20)));
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(new int(30)).At(Timestamp(30)));
|
||||
// input2: floats 5.5, 35.5 at times 5, 35.
|
||||
runner.MutableInputs()->Index(1).packets.push_back(
|
||||
Adopt(new float(5.5)).At(Timestamp(5)));
|
||||
runner.MutableInputs()->Index(1).packets.push_back(
|
||||
Adopt(new float(35.5)).At(Timestamp(35)));
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
// Expected combined_output: 5.5, 10, 20, 30, 35.5 at times 5, 10, 20, 30, 35.
|
||||
const std::vector<Packet>& actual_output = runner.Outputs().Index(0).packets;
|
||||
ASSERT_EQ(actual_output.size(), 5);
|
||||
EXPECT_EQ(actual_output[0].Timestamp(), Timestamp(5));
|
||||
EXPECT_EQ(actual_output[0].Get<float>(), 5.5);
|
||||
|
||||
EXPECT_EQ(actual_output[1].Timestamp(), Timestamp(10));
|
||||
EXPECT_EQ(actual_output[1].Get<int>(), 10);
|
||||
|
||||
EXPECT_EQ(actual_output[2].Timestamp(), Timestamp(20));
|
||||
EXPECT_EQ(actual_output[2].Get<int>(), 20);
|
||||
|
||||
EXPECT_EQ(actual_output[3].Timestamp(), Timestamp(30));
|
||||
EXPECT_EQ(actual_output[3].Get<int>(), 30);
|
||||
|
||||
EXPECT_EQ(actual_output[4].Timestamp(), Timestamp(35));
|
||||
EXPECT_EQ(actual_output[4].Get<float>(), 35.5);
|
||||
}
|
||||
|
||||
// Ensures three streams with differing types can be merged correctly.
|
||||
TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
|
||||
TestMergingThreeStreams) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
input_stream: "input3"
|
||||
output_stream: "combined_output"
|
||||
)"));
|
||||
|
||||
// input1: integer 30 occurring at time 30.
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(new int(30)).At(Timestamp(30)));
|
||||
// input2: float 20.5 occurring at time 20.
|
||||
runner.MutableInputs()->Index(1).packets.push_back(
|
||||
Adopt(new float(20.5)).At(Timestamp(20)));
|
||||
// input3: char 'c' occurring at time 10.
|
||||
runner.MutableInputs()->Index(2).packets.push_back(
|
||||
Adopt(new char('c')).At(Timestamp(10)));
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
// Expected combined_output: 'c', 20.5, 30 at times 10, 20, 30.
|
||||
const std::vector<Packet>& actual_output = runner.Outputs().Index(0).packets;
|
||||
ASSERT_EQ(actual_output.size(), 3);
|
||||
EXPECT_EQ(actual_output[0].Timestamp(), Timestamp(10));
|
||||
EXPECT_EQ(actual_output[0].Get<char>(), 'c');
|
||||
|
||||
EXPECT_EQ(actual_output[1].Timestamp(), Timestamp(20));
|
||||
EXPECT_EQ(actual_output[1].Get<float>(), 20.5);
|
||||
|
||||
EXPECT_EQ(actual_output[2].Timestamp(), Timestamp(30));
|
||||
EXPECT_EQ(actual_output[2].Get<int>(), 30);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -51,7 +51,7 @@ class MuxCalculator : public CalculatorBase {
|
||||
data_input_base_ = cc->Inputs().GetId("INPUT", 0);
|
||||
num_data_inputs_ = cc->Inputs().NumEntries("INPUT");
|
||||
output_ = cc->Outputs().GetId("OUTPUT", 0);
|
||||
cc->SetOffset(mediapipe::TimestampDiff(0));
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/core/packet_cloner_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
|
||||
namespace mediapipe {
|
||||
@@ -39,6 +40,7 @@ namespace mediapipe {
|
||||
// }
|
||||
//
|
||||
// Related:
|
||||
// packet_cloner_calculator.proto: Options for this calculator.
|
||||
// merge_input_streams_calculator.cc: One output stream.
|
||||
// packet_inner_join_calculator.cc: Don't output unless all inputs are new.
|
||||
class PacketClonerCalculator : public CalculatorBase {
|
||||
@@ -54,6 +56,13 @@ class PacketClonerCalculator : public CalculatorBase {
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) final {
|
||||
// Load options.
|
||||
const auto calculator_options =
|
||||
cc->Options<mediapipe::PacketClonerCalculatorOptions>();
|
||||
output_only_when_all_inputs_received_ =
|
||||
calculator_options.output_only_when_all_inputs_received();
|
||||
|
||||
// Parse input streams.
|
||||
tick_signal_index_ = cc->Inputs().NumEntries() - 1;
|
||||
current_.resize(tick_signal_index_);
|
||||
// Pass along the header for each stream if present.
|
||||
@@ -73,8 +82,17 @@ class PacketClonerCalculator : public CalculatorBase {
|
||||
}
|
||||
}
|
||||
|
||||
// Output if the tick signal is non-empty.
|
||||
// Output according to the TICK signal.
|
||||
if (!cc->Inputs().Index(tick_signal_index_).Value().IsEmpty()) {
|
||||
if (output_only_when_all_inputs_received_) {
|
||||
// Return if one of the input is null.
|
||||
for (int i = 0; i < tick_signal_index_; ++i) {
|
||||
if (current_[i].IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
}
|
||||
}
|
||||
// Output each stream.
|
||||
for (int i = 0; i < tick_signal_index_; ++i) {
|
||||
if (!current_[i].IsEmpty()) {
|
||||
cc->Outputs().Index(i).AddPacket(
|
||||
@@ -91,6 +109,7 @@ class PacketClonerCalculator : public CalculatorBase {
|
||||
private:
|
||||
std::vector<Packet> current_;
|
||||
int tick_signal_index_;
|
||||
bool output_only_when_all_inputs_received_;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(PacketClonerCalculator);
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message PacketClonerCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional PacketClonerCalculatorOptions ext = 258872085;
|
||||
}
|
||||
|
||||
// When true, this calculator will drop received TICK packets if any input
|
||||
// stream hasn't received a packet yet.
|
||||
optional bool output_only_when_all_inputs_received = 1 [default = false];
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Calculator that acts like the SQL query:
|
||||
// SELECT *
|
||||
// FROM packets_on_stream1 AS packet1
|
||||
// INNER JOIN packets_on_stream2 AS packet2
|
||||
// ON packet1.timestamp = packet2.timestamp
|
||||
//
|
||||
// In other words, it only emits and forwards packets if all input streams are
|
||||
// not empty.
|
||||
//
|
||||
// Intended for use with FixedSizeInputStreamHandler.
|
||||
//
|
||||
// Related:
|
||||
// packet_cloner_calculator.cc: Repeats last-seen packets from empty inputs.
|
||||
class PacketInnerJoinCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
int num_streams_;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(PacketInnerJoinCalculator);
|
||||
|
||||
::mediapipe::Status PacketInnerJoinCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().NumEntries() == cc->Outputs().NumEntries())
|
||||
<< "The number of input and output streams must match.";
|
||||
const int num_streams = cc->Inputs().NumEntries();
|
||||
for (int i = 0; i < num_streams; ++i) {
|
||||
cc->Inputs().Index(i).SetAny();
|
||||
cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status PacketInnerJoinCalculator::Open(CalculatorContext* cc) {
|
||||
num_streams_ = cc->Inputs().NumEntries();
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status PacketInnerJoinCalculator::Process(CalculatorContext* cc) {
|
||||
for (int i = 0; i < num_streams_; ++i) {
|
||||
if (cc->Inputs().Index(i).Value().IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < num_streams_; ++i) {
|
||||
cc->Outputs().Index(i).AddPacket(cc->Inputs().Index(i).Value());
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
inline Packet PacketFrom(int i) { return Adopt(new int(i)).At(Timestamp(i)); }
|
||||
|
||||
TEST(PacketInnerJoinCalculatorTest, AllMatching) {
|
||||
// Test case.
|
||||
const std::vector<int> packets_on_stream1 = {0, 1, 2, 3};
|
||||
const std::vector<int> packets_on_stream2 = {0, 1, 2, 3};
|
||||
// Run.
|
||||
CalculatorRunner runner("PacketInnerJoinCalculator", "", 2, 2, 0);
|
||||
for (int packet_load : packets_on_stream1) {
|
||||
runner.MutableInputs()->Index(0).packets.push_back(PacketFrom(packet_load));
|
||||
}
|
||||
for (int packet_load : packets_on_stream2) {
|
||||
runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load));
|
||||
}
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
// Check.
|
||||
const std::vector<int> expected = {0, 1, 2, 3};
|
||||
ASSERT_EQ(expected.size(), runner.Outputs().Index(0).packets.size());
|
||||
ASSERT_EQ(expected.size(), runner.Outputs().Index(1).packets.size());
|
||||
for (int i = 0; i < expected.size(); ++i) {
|
||||
const Packet packet1 = runner.Outputs().Index(0).packets[i];
|
||||
EXPECT_EQ(expected[i], packet1.Get<int>());
|
||||
EXPECT_EQ(expected[i], packet1.Timestamp().Value());
|
||||
const Packet packet2 = runner.Outputs().Index(1).packets[i];
|
||||
EXPECT_EQ(expected[i], packet2.Get<int>());
|
||||
EXPECT_EQ(expected[i], packet2.Timestamp().Value());
|
||||
}
|
||||
}
|
||||
|
||||
TEST(PacketInnerJoinCalculatorTest, NoneMatching) {
|
||||
// Test case.
|
||||
const std::vector<int> packets_on_stream1 = {0, 2};
|
||||
const std::vector<int> packets_on_stream2 = {1, 3};
|
||||
// Run.
|
||||
CalculatorRunner runner("PacketInnerJoinCalculator", "", 2, 2, 0);
|
||||
for (int packet_load : packets_on_stream1) {
|
||||
runner.MutableInputs()->Index(0).packets.push_back(PacketFrom(packet_load));
|
||||
}
|
||||
for (int packet_load : packets_on_stream2) {
|
||||
runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load));
|
||||
}
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
// Check.
|
||||
EXPECT_TRUE(runner.Outputs().Index(0).packets.empty());
|
||||
EXPECT_TRUE(runner.Outputs().Index(1).packets.empty());
|
||||
}
|
||||
|
||||
TEST(PacketInnerJoinCalculatorTest, SomeMatching) {
|
||||
// Test case.
|
||||
const std::vector<int> packets_on_stream1 = {0, 1, 2, 3, 4, 6};
|
||||
const std::vector<int> packets_on_stream2 = {0, 2, 4, 5, 6};
|
||||
// Run.
|
||||
CalculatorRunner runner("PacketInnerJoinCalculator", "", 2, 2, 0);
|
||||
for (int packet_load : packets_on_stream1) {
|
||||
runner.MutableInputs()->Index(0).packets.push_back(PacketFrom(packet_load));
|
||||
}
|
||||
for (int packet_load : packets_on_stream2) {
|
||||
runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load));
|
||||
}
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
// Check.
|
||||
const std::vector<int> expected = {0, 2, 4, 6};
|
||||
ASSERT_EQ(expected.size(), runner.Outputs().Index(0).packets.size());
|
||||
ASSERT_EQ(expected.size(), runner.Outputs().Index(1).packets.size());
|
||||
for (int i = 0; i < expected.size(); ++i) {
|
||||
const Packet packet1 = runner.Outputs().Index(0).packets[i];
|
||||
EXPECT_EQ(expected[i], packet1.Get<int>());
|
||||
EXPECT_EQ(expected[i], packet1.Timestamp().Value());
|
||||
const Packet packet2 = runner.Outputs().Index(1).packets[i];
|
||||
EXPECT_EQ(expected[i], packet2.Get<int>());
|
||||
EXPECT_EQ(expected[i], packet2.Timestamp().Value());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -12,25 +12,17 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cstdlib>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include "mediapipe/calculators/core/packet_resampler_calculator.h"
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/deps/mathutil.h"
|
||||
#include "mediapipe/framework/deps/random_base.h"
|
||||
#include "mediapipe/framework/formats/video_stream_header.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
#include "mediapipe/framework/tool/options_util.h"
|
||||
#include <memory>
|
||||
|
||||
namespace {
|
||||
// Reflect an integer against the lower and upper bound of an interval.
|
||||
int64 ReflectBetween(int64 ts, int64 ts_min, int64 ts_max) {
|
||||
if (ts < ts_min) return 2 * ts_min - ts - 1;
|
||||
if (ts >= ts_max) return 2 * ts_max - ts - 1;
|
||||
return ts;
|
||||
}
|
||||
|
||||
// Creates a secure random number generator for use in ProcessWithJitter.
|
||||
// If no secure random number generator can be constructed, the jitter
|
||||
@@ -45,114 +37,7 @@ std::unique_ptr<RandomBase> CreateSecureRandom(const std::string& seed) {
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// This calculator is used to normalize the frequency of the packets
|
||||
// out of a stream. Given a desired frame rate, packets are going to be
|
||||
// removed or added to achieve it.
|
||||
//
|
||||
// The jitter feature is disabled by default. To enable it, you need to
|
||||
// implement CreateSecureRandom(const std::string&).
|
||||
//
|
||||
// The data stream may be either specified as the only stream (by index)
|
||||
// or as the stream with tag "DATA".
|
||||
//
|
||||
// The input and output streams may be accompanied by a VIDEO_HEADER
|
||||
// stream. This stream includes a VideoHeader at Timestamp::PreStream().
|
||||
// The input VideoHeader on the VIDEO_HEADER stream will always be updated
|
||||
// with the resampler frame rate no matter what the options value for
|
||||
// output_header is before being output on the output VIDEO_HEADER stream.
|
||||
// If the input VideoHeader is not available, then only the frame rate
|
||||
// value will be set in the output.
|
||||
//
|
||||
// Related:
|
||||
// packet_downsampler_calculator.cc: skips packets regardless of timestamps.
|
||||
class PacketResamplerCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Logic for Process() when jitter_ != 0.0.
|
||||
::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
|
||||
|
||||
// Logic for Process() when jitter_ == 0.0.
|
||||
::mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);
|
||||
|
||||
// Given the current count of periods that have passed, this returns
|
||||
// the next valid timestamp of the middle point of the next period:
|
||||
// if count is 0, it returns the first_timestamp_.
|
||||
// if count is 1, it returns the first_timestamp_ + period (corresponding
|
||||
// to the first tick using exact fps)
|
||||
// e.g. for frame_rate=30 and first_timestamp_=0:
|
||||
// 0: 0
|
||||
// 1: 33333
|
||||
// 2: 66667
|
||||
// 3: 100000
|
||||
//
|
||||
// Can only be used if jitter_ equals zero.
|
||||
Timestamp PeriodIndexToTimestamp(int64 index) const;
|
||||
|
||||
// Given a Timestamp, finds the closest sync Timestamp based on
|
||||
// first_timestamp_ and the desired fps.
|
||||
//
|
||||
// Can only be used if jitter_ equals zero.
|
||||
int64 TimestampToPeriodIndex(Timestamp timestamp) const;
|
||||
|
||||
// Outputs a packet if it is in range (start_time_, end_time_).
|
||||
void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
|
||||
|
||||
// The timestamp of the first packet received.
|
||||
Timestamp first_timestamp_;
|
||||
|
||||
// Number of frames per second (desired output frequency).
|
||||
double frame_rate_;
|
||||
|
||||
// Inverse of frame_rate_.
|
||||
int64 frame_time_usec_;
|
||||
|
||||
// Number of periods that have passed (= #packets sent to the output).
|
||||
//
|
||||
// Can only be used if jitter_ equals zero.
|
||||
int64 period_count_;
|
||||
|
||||
// The last packet that was received.
|
||||
Packet last_packet_;
|
||||
|
||||
VideoHeader video_header_;
|
||||
// The "DATA" input stream.
|
||||
CollectionItemId input_data_id_;
|
||||
// The "DATA" output stream.
|
||||
CollectionItemId output_data_id_;
|
||||
|
||||
// Indicator whether to flush last packet even if its timestamp is greater
|
||||
// than the final stream timestamp. Set to false when jitter_ is non-zero.
|
||||
bool flush_last_packet_;
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
double jitter_ = 0.0;
|
||||
Timestamp next_output_timestamp_;
|
||||
|
||||
// If specified, output timestamps are aligned with base_timestamp.
|
||||
// Otherwise, they are aligned with the first input timestamp.
|
||||
Timestamp base_timestamp_;
|
||||
|
||||
// If specified, only outputs at/after start_time are included.
|
||||
Timestamp start_time_;
|
||||
|
||||
// If specified, only outputs before end_time are included.
|
||||
Timestamp end_time_;
|
||||
|
||||
// If set, the output timestamps nearest to start_time and end_time
|
||||
// are included in the output, even if the nearest timestamp is not
|
||||
// between start_time and end_time.
|
||||
bool round_limits_;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(PacketResamplerCalculator);
|
||||
|
||||
namespace {
|
||||
// Returns a TimestampDiff (assuming microseconds) corresponding to the
|
||||
// given time in seconds.
|
||||
@@ -203,6 +88,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
||||
|
||||
flush_last_packet_ = resampler_options.flush_last_packet();
|
||||
jitter_ = resampler_options.jitter();
|
||||
jitter_with_reflection_ = resampler_options.jitter_with_reflection();
|
||||
|
||||
input_data_id_ = cc->Inputs().GetId("DATA", 0);
|
||||
if (!input_data_id_.IsValid()) {
|
||||
@@ -233,6 +119,9 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
||||
<< Timestamp::kTimestampUnitsPerSecond;
|
||||
|
||||
frame_time_usec_ = static_cast<int64>(1000000.0 / frame_rate_);
|
||||
jitter_usec_ = static_cast<int64>(1000000.0 * jitter_ / frame_rate_);
|
||||
RET_CHECK_LE(jitter_usec_, frame_time_usec_);
|
||||
|
||||
video_header_.frame_rate = frame_rate_;
|
||||
|
||||
if (resampler_options.output_header() !=
|
||||
@@ -272,7 +161,10 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
||||
"SecureRandom is not available. With \"jitter\" specified, "
|
||||
"PacketResamplerCalculator processing cannot proceed.");
|
||||
}
|
||||
packet_reservoir_random_ = CreateSecureRandom(seed);
|
||||
}
|
||||
packet_reservoir_ =
|
||||
std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
@@ -287,14 +179,56 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
||||
}
|
||||
}
|
||||
if (jitter_ != 0.0 && random_ != nullptr) {
|
||||
RETURN_IF_ERROR(ProcessWithJitter(cc));
|
||||
// Packet reservior is used to make sure there's an output for every period,
|
||||
// e.g. partial period at the end of the stream.
|
||||
if (packet_reservoir_->IsEnabled() &&
|
||||
(first_timestamp_ == Timestamp::Unset() ||
|
||||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
|
||||
auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
|
||||
packet_reservoir_->AddSample(curr_packet);
|
||||
}
|
||||
MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
|
||||
} else {
|
||||
RETURN_IF_ERROR(ProcessWithoutJitter(cc));
|
||||
MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
|
||||
}
|
||||
last_packet_ = cc->Inputs().Get(input_data_id_).Value();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
|
||||
next_output_timestamp_min_ = first_timestamp_;
|
||||
if (jitter_with_reflection_) {
|
||||
next_output_timestamp_ =
|
||||
first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
|
||||
return;
|
||||
}
|
||||
next_output_timestamp_ =
|
||||
first_timestamp_ + frame_time_usec_ * random_->RandFloat();
|
||||
}
|
||||
|
||||
void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
|
||||
packet_reservoir_->Clear();
|
||||
if (jitter_with_reflection_) {
|
||||
next_output_timestamp_min_ += frame_time_usec_;
|
||||
Timestamp next_output_timestamp_max_ =
|
||||
next_output_timestamp_min_ + frame_time_usec_;
|
||||
|
||||
next_output_timestamp_ += frame_time_usec_ +
|
||||
random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
|
||||
jitter_usec_;
|
||||
next_output_timestamp_ = Timestamp(ReflectBetween(
|
||||
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
||||
next_output_timestamp_max_.Value()));
|
||||
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
|
||||
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
|
||||
return;
|
||||
}
|
||||
packet_reservoir_->Disable();
|
||||
next_output_timestamp_ +=
|
||||
frame_time_usec_ *
|
||||
((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
|
||||
}
|
||||
|
||||
::mediapipe::Status PacketResamplerCalculator::ProcessWithJitter(
|
||||
CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
@@ -302,29 +236,37 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
||||
|
||||
if (first_timestamp_ == Timestamp::Unset()) {
|
||||
first_timestamp_ = cc->InputTimestamp();
|
||||
next_output_timestamp_ =
|
||||
first_timestamp_ + frame_time_usec_ * random_->RandFloat();
|
||||
InitializeNextOutputTimestampWithJitter();
|
||||
if (first_timestamp_ == next_output_timestamp_) {
|
||||
OutputWithinLimits(
|
||||
cc,
|
||||
cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
LOG_IF(WARNING, frame_time_usec_ <
|
||||
(cc->InputTimestamp() - last_packet_.Timestamp()).Value())
|
||||
<< "Adding jitter is meaningless when upsampling.";
|
||||
if (frame_time_usec_ <
|
||||
(cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
|
||||
LOG_FIRST_N(WARNING, 2)
|
||||
<< "Adding jitter is not very useful when upsampling.";
|
||||
}
|
||||
|
||||
const int64 curr_diff =
|
||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||
const int64 last_diff =
|
||||
(next_output_timestamp_ - last_packet_.Timestamp()).Value();
|
||||
if (curr_diff * last_diff > 0) {
|
||||
return ::mediapipe::OkStatus();
|
||||
while (true) {
|
||||
const int64 last_diff =
|
||||
(next_output_timestamp_ - last_packet_.Timestamp()).Value();
|
||||
RET_CHECK_GT(last_diff, 0);
|
||||
const int64 curr_diff =
|
||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||
if (curr_diff > 0) {
|
||||
break;
|
||||
}
|
||||
OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
|
||||
? last_packet_
|
||||
: cc->Inputs().Get(input_data_id_).Value())
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
}
|
||||
OutputWithinLimits(cc, (std::abs(curr_diff) > std::abs(last_diff)
|
||||
? last_packet_
|
||||
: cc->Inputs().Get(input_data_id_).Value())
|
||||
.At(next_output_timestamp_));
|
||||
next_output_timestamp_ +=
|
||||
frame_time_usec_ *
|
||||
((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
@@ -405,6 +347,9 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
||||
OutputWithinLimits(cc,
|
||||
last_packet_.At(PeriodIndexToTimestamp(period_count_)));
|
||||
}
|
||||
if (!packet_reservoir_->IsEmpty()) {
|
||||
OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
|
||||
|
||||
#include <cstdlib>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/deps/mathutil.h"
|
||||
#include "mediapipe/framework/deps/random_base.h"
|
||||
#include "mediapipe/framework/formats/video_stream_header.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
#include "mediapipe/framework/tool/options_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class PacketReservoir {
|
||||
public:
|
||||
PacketReservoir(RandomBase* rng) : rng_(rng) {}
|
||||
// Replace candidate with current packet with 1/count_ probability.
|
||||
void AddSample(Packet sample) {
|
||||
if (rng_->UnbiasedUniform(++count_) == 0) {
|
||||
reservoir_ = sample;
|
||||
}
|
||||
}
|
||||
bool IsEnabled() { return rng_ && enabled_; }
|
||||
void Disable() {
|
||||
if (enabled_) enabled_ = false;
|
||||
}
|
||||
void Clear() { count_ = 0; }
|
||||
bool IsEmpty() { return count_ == 0; }
|
||||
Packet GetSample() { return reservoir_; }
|
||||
|
||||
private:
|
||||
RandomBase* rng_;
|
||||
bool enabled_ = true;
|
||||
int32 count_ = 0;
|
||||
Packet reservoir_;
|
||||
};
|
||||
|
||||
// This calculator is used to normalize the frequency of the packets
|
||||
// out of a stream. Given a desired frame rate, packets are going to be
|
||||
// removed or added to achieve it.
|
||||
//
|
||||
// If jitter_ is specified:
|
||||
// - The first packet is chosen randomly (uniform distribution) among frames
|
||||
// that correspond to timestamps [0, 1/frame_rate). Let the chosen packet
|
||||
// correspond to timestamp t.
|
||||
// - The next packet is chosen randomly (uniform distribution) among frames
|
||||
// that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
|
||||
// - if jitter_with_reflection_ is true, the timestamp will be reflected
|
||||
// against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
|
||||
// so that its marginal distribution is uniform within this interval.
|
||||
// In the formula, t_0 is the timestamp of the first sampled
|
||||
// packet, and the k is the packet index.
|
||||
// See paper (https://arxiv.org/abs/2002.01147) for details.
|
||||
// - t is updated and the process is repeated.
|
||||
// - Note that seed is specified as input side packet for reproducibility of
|
||||
// the resampling. For Cloud ML Video Intelligence API, the hash of the
|
||||
// input video should serve this purpose. For YouTube, either video ID or
|
||||
// content hex ID of the input video should do.
|
||||
//
|
||||
// If jitter_ is not specified:
|
||||
// - The first packet defines the first_timestamp of the output stream,
|
||||
// so it is always emitted.
|
||||
// - If more packets are emitted, they will have timestamp equal to
|
||||
// round(first_timestamp + k * period) , where k is a positive
|
||||
// integer and the period is defined by the frame rate.
|
||||
// Example: first_timestamp=0, fps=30, then the output stream
|
||||
// will have timestamps: 0, 33333, 66667, 100000, etc...
|
||||
// - The packets selected for the output stream are the ones closer
|
||||
// to the exact middle point (33333.33, 66666.67 in our previous
|
||||
// example). In case of ties, later packets are chosen.
|
||||
// - 'Empty' periods happen when there are no packets for a long time
|
||||
// (greater than a period). In this case, we send a copy of the last
|
||||
// packet received before the empty period.
|
||||
// The jitter feature is disabled by default. To enable it, you need to
|
||||
// implement CreateSecureRandom(const std::string&).
|
||||
//
|
||||
// The data stream may be either specified as the only stream (by index)
|
||||
// or as the stream with tag "DATA".
|
||||
//
|
||||
// The input and output streams may be accompanied by a VIDEO_HEADER
|
||||
// stream. This stream includes a VideoHeader at Timestamp::PreStream().
|
||||
// The input VideoHeader on the VIDEO_HEADER stream will always be updated
|
||||
// with the resampler frame rate no matter what the options value for
|
||||
// output_header is before being output on the output VIDEO_HEADER stream.
|
||||
// If the input VideoHeader is not available, then only the frame rate
|
||||
// value will be set in the output.
|
||||
//
|
||||
// Related:
|
||||
// packet_downsampler_calculator.cc: skips packets regardless of timestamps.
|
||||
class PacketResamplerCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Calculates the first sampled timestamp that incorporates a jittering
|
||||
// offset.
|
||||
void InitializeNextOutputTimestampWithJitter();
|
||||
// Calculates the next sampled timestamp that incorporates a jittering offset.
|
||||
void UpdateNextOutputTimestampWithJitter();
|
||||
|
||||
// Logic for Process() when jitter_ != 0.0.
|
||||
::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
|
||||
|
||||
// Logic for Process() when jitter_ == 0.0.
|
||||
::mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);
|
||||
|
||||
// Given the current count of periods that have passed, this returns
|
||||
// the next valid timestamp of the middle point of the next period:
|
||||
// if count is 0, it returns the first_timestamp_.
|
||||
// if count is 1, it returns the first_timestamp_ + period (corresponding
|
||||
// to the first tick using exact fps)
|
||||
// e.g. for frame_rate=30 and first_timestamp_=0:
|
||||
// 0: 0
|
||||
// 1: 33333
|
||||
// 2: 66667
|
||||
// 3: 100000
|
||||
//
|
||||
// Can only be used if jitter_ equals zero.
|
||||
Timestamp PeriodIndexToTimestamp(int64 index) const;
|
||||
|
||||
// Given a Timestamp, finds the closest sync Timestamp based on
|
||||
// first_timestamp_ and the desired fps.
|
||||
//
|
||||
// Can only be used if jitter_ equals zero.
|
||||
int64 TimestampToPeriodIndex(Timestamp timestamp) const;
|
||||
|
||||
// Outputs a packet if it is in range (start_time_, end_time_).
|
||||
void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
|
||||
|
||||
// The timestamp of the first packet received.
|
||||
Timestamp first_timestamp_;
|
||||
|
||||
// Number of frames per second (desired output frequency).
|
||||
double frame_rate_;
|
||||
|
||||
// Inverse of frame_rate_.
|
||||
int64 frame_time_usec_;
|
||||
|
||||
// Number of periods that have passed (= #packets sent to the output).
|
||||
//
|
||||
// Can only be used if jitter_ equals zero.
|
||||
int64 period_count_;
|
||||
|
||||
// The last packet that was received.
|
||||
Packet last_packet_;
|
||||
|
||||
VideoHeader video_header_;
|
||||
// The "DATA" input stream.
|
||||
CollectionItemId input_data_id_;
|
||||
// The "DATA" output stream.
|
||||
CollectionItemId output_data_id_;
|
||||
|
||||
// Indicator whether to flush last packet even if its timestamp is greater
|
||||
// than the final stream timestamp. Set to false when jitter_ is non-zero.
|
||||
bool flush_last_packet_;
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
double jitter_ = 0.0;
|
||||
bool jitter_with_reflection_;
|
||||
int64 jitter_usec_;
|
||||
Timestamp next_output_timestamp_;
|
||||
// If jittering_with_reflection_ is true, next_output_timestamp_ will be
|
||||
// kept within the interval
|
||||
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
|
||||
Timestamp next_output_timestamp_min_;
|
||||
|
||||
// If specified, output timestamps are aligned with base_timestamp.
|
||||
// Otherwise, they are aligned with the first input timestamp.
|
||||
Timestamp base_timestamp_;
|
||||
|
||||
// If specified, only outputs at/after start_time are included.
|
||||
Timestamp start_time_;
|
||||
|
||||
// If specified, only outputs before end_time are included.
|
||||
Timestamp end_time_;
|
||||
|
||||
// If set, the output timestamps nearest to start_time and end_time
|
||||
// are included in the output, even if the nearest timestamp is not
|
||||
// between start_time and end_time.W
|
||||
bool round_limits_;
|
||||
|
||||
// packet reservior used for sampling random packet out of partial
|
||||
// period when jitter is enabled
|
||||
std::unique_ptr<PacketReservoir> packet_reservoir_;
|
||||
// random number generator used in packet_reservior_.
|
||||
std::unique_ptr<RandomBase> packet_reservoir_random_;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
#endif // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
|
||||
@@ -66,6 +66,7 @@ message PacketResamplerCalculatorOptions {
|
||||
// pseudo-random number generator does its job and the number of frames is
|
||||
// sufficiently large, the average frame rate will be close to this value.
|
||||
optional double jitter = 4;
|
||||
optional bool jitter_with_reflection = 9 [default = false];
|
||||
|
||||
// If specified, output timestamps are aligned with base_timestamp.
|
||||
// Otherwise, they are aligned with the first input timestamp.
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/packet_resampler_calculator.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
@@ -29,7 +31,6 @@
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
// A simple version of CalculatorRunner with built-in convenience
|
||||
// methods for setting inputs from a vector and checking outputs
|
||||
// against expected outputs (both timestamps and contents).
|
||||
@@ -103,7 +104,7 @@ TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,7 +115,7 @@ TEST(PacketResamplerCalculatorTest, SinglePacketInStream) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0}, {0});
|
||||
}
|
||||
|
||||
@@ -124,7 +125,7 @@ TEST(PacketResamplerCalculatorTest, SinglePacketInStream) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({1000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({1000}, {1000});
|
||||
}
|
||||
|
||||
@@ -134,7 +135,7 @@ TEST(PacketResamplerCalculatorTest, SinglePacketInStream) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({16668});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({16668}, {16668});
|
||||
}
|
||||
}
|
||||
@@ -146,7 +147,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0, 16666});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0}, {0});
|
||||
}
|
||||
|
||||
@@ -156,7 +157,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0, 16667});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 16667}, {0, 33333});
|
||||
}
|
||||
|
||||
@@ -166,7 +167,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0, 49999});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 49999}, {0, 33333});
|
||||
}
|
||||
|
||||
@@ -176,7 +177,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0, 50000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 0, 50000}, {0, 33333, 66667});
|
||||
}
|
||||
|
||||
@@ -186,7 +187,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({2000, 118666});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({2000, 2000, 2000, 118666},
|
||||
{2000, 35333, 68667, 102000});
|
||||
}
|
||||
@@ -197,7 +198,7 @@ TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepoints) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0, 33333, 66667, 100000, 133333, 166667, 200000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps(
|
||||
{0, 33333, 66667, 100000, 133333, 166667, 200000},
|
||||
{0, 33333, 66667, 100000, 133333, 166667, 200000});
|
||||
@@ -210,7 +211,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriods) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0, 16666, 16667, 20000, 33300, 49999, 50000, 66600});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 33300, 66600}, {0, 33333, 66667});
|
||||
}
|
||||
|
||||
@@ -222,7 +223,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0, 5000, 16666, 83334});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 16666, 16666, 83334},
|
||||
{0, 33333, 66667, 100000});
|
||||
}
|
||||
@@ -232,7 +233,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0, 16666, 16667, 25000, 33000, 35000, 135000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 33000, 35000, 35000, 135000},
|
||||
{0, 33333, 66667, 100000, 133333});
|
||||
}
|
||||
@@ -242,7 +243,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({0, 15000, 32000, 49999, 150000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 32000, 49999, 49999, 49999, 150000},
|
||||
{0, 33333, 66667, 100000, 133333, 166667});
|
||||
}
|
||||
@@ -255,7 +256,7 @@ TEST(PacketResamplerCalculatorTest, SuperHighFrameRate) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:500000}");
|
||||
runner.SetInput({0, 10, 13});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 0, 0, 0, 0, 10, 10, 13},
|
||||
{0, 2, 4, 6, 8, 10, 12, 14});
|
||||
}
|
||||
@@ -266,7 +267,7 @@ TEST(PacketResamplerCalculatorTest, SuperHighFrameRate) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:1000000}");
|
||||
runner.SetInput({0, 10, 13});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps(
|
||||
{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 13},
|
||||
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13});
|
||||
@@ -280,7 +281,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({-200, -20, 16466});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({-200}, {-200});
|
||||
}
|
||||
|
||||
@@ -290,7 +291,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({-200, -20, 16467});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({-200, 16467}, {-200, 33133});
|
||||
}
|
||||
|
||||
@@ -300,7 +301,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({-500, 66667});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({-500, -500, 66667}, {-500, 32833, 66167});
|
||||
}
|
||||
|
||||
@@ -310,7 +311,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({-50000, -33334, 33334});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({-50000, -33334, -33334, 33334},
|
||||
{-50000, -16667, 16667, 50000});
|
||||
}
|
||||
@@ -323,7 +324,7 @@ TEST(PacketResamplerCalculatorTest, ExactFramesPerSecond) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:50}");
|
||||
runner.SetInput({0, 9999, 29999});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 29999}, {0, 20000});
|
||||
}
|
||||
|
||||
@@ -333,7 +334,7 @@ TEST(PacketResamplerCalculatorTest, ExactFramesPerSecond) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:50}");
|
||||
runner.SetInput({0, 10000, 50000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 10000, 10000, 50000},
|
||||
{0, 20000, 40000, 60000});
|
||||
}
|
||||
@@ -347,7 +348,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
|
||||
"{frame_rate:50, output_header:UPDATE_VIDEO_HEADER}");
|
||||
runner.SetInput({0, 10000, 30000, 50000, 60000});
|
||||
runner.SetVideoHeader(50.0);
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 10000, 30000, 60000},
|
||||
{0, 20000, 40000, 60000});
|
||||
runner.CheckVideoHeader(50.0);
|
||||
@@ -360,7 +361,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
|
||||
"{frame_rate:50, output_header:UPDATE_VIDEO_HEADER}");
|
||||
runner.SetInput({0, 5000, 10010, 15001, 19990});
|
||||
runner.SetVideoHeader(200.0);
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 19990}, {0, 20000});
|
||||
runner.CheckVideoHeader(50.0);
|
||||
}
|
||||
@@ -372,7 +373,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
|
||||
"{frame_rate:50, output_header:PASS_HEADER}");
|
||||
runner.SetInput({0, 5000, 10010, 15001, 19990});
|
||||
runner.SetVideoHeader(200.0);
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({0, 19990}, {0, 20000});
|
||||
runner.CheckVideoHeader(200.0);
|
||||
}
|
||||
@@ -404,7 +405,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
||||
->Tag("VIDEO_HEADER")
|
||||
.packets.push_back(
|
||||
Adopt(new VideoHeader(video_header_in)).At(Timestamp::PreStream()));
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
ASSERT_EQ(1, runner.Outputs().Tag("VIDEO_HEADER").packets.size());
|
||||
EXPECT_EQ(Timestamp::PreStream(),
|
||||
@@ -424,7 +425,7 @@ TEST(PacketResamplerCalculatorTest, FlushLastPacketWithoutRound) {
|
||||
frame_rate: 1
|
||||
})");
|
||||
runner.SetInput({0, 333333, 666667, 1000000, 1333333});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
// 1333333 is not emitted as 2000000, because it does not round to 2000000.
|
||||
runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000});
|
||||
}
|
||||
@@ -435,7 +436,7 @@ TEST(PacketResamplerCalculatorTest, FlushLastPacketWithRound) {
|
||||
frame_rate: 1
|
||||
})");
|
||||
runner.SetInput({0, 333333, 666667, 1000000, 1333333, 1666667});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
// 1666667 is emitted as 2000000, because it rounds to 2000000.
|
||||
runner.CheckOutputTimestamps({0, 1000000, 1666667}, {0, 1000000, 2000000});
|
||||
}
|
||||
@@ -447,7 +448,7 @@ TEST(PacketResamplerCalculatorTest, DoNotFlushLastPacketWithoutRound) {
|
||||
flush_last_packet: false
|
||||
})");
|
||||
runner.SetInput({0, 333333, 666667, 1000000, 1333333});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
// 1333333 is not emitted no matter what; see FlushLastPacketWithoutRound.
|
||||
runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000});
|
||||
}
|
||||
@@ -459,7 +460,7 @@ TEST(PacketResamplerCalculatorTest, DoNotFlushLastPacketWithRound) {
|
||||
flush_last_packet: false
|
||||
})");
|
||||
runner.SetInput({0, 333333, 666667, 1000000, 1333333, 1666667});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
// 1666667 is not emitted due to flush_last_packet: false.
|
||||
runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000});
|
||||
}
|
||||
@@ -473,7 +474,7 @@ TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepointsAligned) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({33111, 66667, 100000, 133333, 166667, 200000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({33111, 66667, 100000, 133333, 166667, 200000},
|
||||
{33111, 66444, 99778, 133111, 166444, 199778});
|
||||
}
|
||||
@@ -484,7 +485,7 @@ TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepointsAligned) {
|
||||
"{frame_rate:30 "
|
||||
"base_timestamp:0}");
|
||||
runner.SetInput({33111, 66667, 100000, 133333, 166667, 200000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps(
|
||||
{33111, 66667, 100000, 133333, 166667, 200000},
|
||||
{33333, 66666, 100000, 133333, 166666, 200000});
|
||||
@@ -499,7 +500,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({-222, 16666, 16667, 20000, 33300, 49999, 50000, 66600});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({-222, 33300, 66600}, {-222, 33111, 66445});
|
||||
}
|
||||
{
|
||||
@@ -509,7 +510,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) {
|
||||
"{frame_rate:30 "
|
||||
"base_timestamp:900011}");
|
||||
runner.SetInput({-222, 16666, 16667, 20000, 33300, 49999, 50000, 66600});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({-222, 33300, 66600}, {11, 33344, 66678});
|
||||
}
|
||||
{
|
||||
@@ -521,7 +522,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) {
|
||||
"base_timestamp:11}");
|
||||
runner.SetInput(
|
||||
{899888, 916666, 916667, 920000, 933300, 949999, 950000, 966600});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({899888, 933300, 966600},
|
||||
{900011, 933344, 966678});
|
||||
}
|
||||
@@ -536,7 +537,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacketAligned) {
|
||||
"[mediapipe.PacketResamplerCalculatorOptions.ext]: "
|
||||
"{frame_rate:30}");
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000},
|
||||
{-222, 33111, 66445, 99778, 133111, 166445});
|
||||
}
|
||||
@@ -547,7 +548,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacketAligned) {
|
||||
"{frame_rate:30 "
|
||||
"base_timestamp:0}");
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000},
|
||||
{0, 33333, 66667, 100000, 133333, 166667});
|
||||
}
|
||||
@@ -565,7 +566,7 @@ TEST(PacketResamplerCalculatorTest, FirstInputAfterMiddlepointAligned) {
|
||||
"{frame_rate:30 "
|
||||
"base_timestamp:0}");
|
||||
runner.SetInput({66667, 100020, 133333, 166667});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({66667, 100020, 133333, 166667},
|
||||
{66667, 100000, 133334, 166667});
|
||||
}
|
||||
@@ -582,7 +583,7 @@ TEST(PacketResamplerCalculatorTest, FirstInputAfterMiddlepointAligned) {
|
||||
"{frame_rate:30 "
|
||||
"base_timestamp:0}");
|
||||
runner.SetInput({100020, 133333, 166667});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({100020, 133333, 166667},
|
||||
{100000, 133333, 166667});
|
||||
}
|
||||
@@ -596,7 +597,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
|
||||
"{frame_rate:30 "
|
||||
"base_timestamp:0}");
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000},
|
||||
{0, 33333, 66667, 100000, 133333, 166667});
|
||||
}
|
||||
@@ -609,7 +610,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
|
||||
"start_time:40000 "
|
||||
"end_time:160000}");
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({49999, 49999, 49999},
|
||||
{66667, 100000, 133333});
|
||||
}
|
||||
@@ -624,7 +625,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
|
||||
"end_time:160000 "
|
||||
"round_limits:true}");
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
runner.CheckOutputTimestamps({32000, 49999, 49999, 49999, 150000},
|
||||
{33333, 66667, 100000, 133333, 166667});
|
||||
}
|
||||
@@ -654,7 +655,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
})"));
|
||||
runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
EXPECT_EQ(6, runner.Outputs().Index(0).packets.size());
|
||||
}
|
||||
{
|
||||
@@ -670,7 +671,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
|
||||
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MEDIAPIPE_ASSERT_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
EXPECT_EQ(6, runner.Outputs().Index(0).packets.size());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Declaration of PacketThinnerCalculator.
|
||||
|
||||
#include <cmath> // for ceil
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/video_stream_header.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
const double kTimebaseUs = 1000000; // Microseconds.
|
||||
const char* const kPeriodTag = "PERIOD";
|
||||
} // namespace
|
||||
|
||||
// This calculator is used to thin an input stream of Packets.
|
||||
// An example application would be to sample decoded frames of video
|
||||
// at a coarser temporal resolution. Unless otherwise stated, all
|
||||
// timestamps are in units of microseconds.
|
||||
//
|
||||
// Thinning can be accomplished in one of two ways:
|
||||
// 1) asynchronous thinning (known below as async):
|
||||
// Algorithm does not rely on a master clock and is parameterized only
|
||||
// by a single option -- the period. Once a packet is emitted, the
|
||||
// thinner will discard subsequent packets for the duration of the period
|
||||
// [Analogous to a refractory period during which packet emission is
|
||||
// suppressed.]
|
||||
// Packets arriving before start_time are discarded, as are packets
|
||||
// arriving at or after end_time.
|
||||
// 2) synchronous thinning (known below as sync):
|
||||
// There are two variants of this algorithm, both parameterized by a
|
||||
// start_time and a period. As in (1), packets arriving before start_time
|
||||
// or at/after end_time are discarded. Otherwise, at most one packet is
|
||||
// emitted during a period, centered at timestamps generated by the
|
||||
// expression:
|
||||
// start_time + i * period [where i is a non-negative integer]
|
||||
// During each period, the packet closest to the generated timestamp is
|
||||
// emitted (latest in the case of ties). In the first variant
|
||||
// (sync_output_timestamps = true), the emitted packet is output at the
|
||||
// generated timestamp. In the second variant, the packet is output at
|
||||
// its original timestamp. Both variants emit exactly the same packets,
|
||||
// but at different timestamps.
|
||||
//
|
||||
// Thinning period can be provided in the calculator options or via a
|
||||
// side packet with the tag "PERIOD".
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "PacketThinnerCalculator"
|
||||
// input_stream: "signal"
|
||||
// output_stream: "output"
|
||||
// options {
|
||||
// [mediapipe.PacketThinnerCalculatorOptions.ext] {
|
||||
// thinner_type: SYNC
|
||||
// period: 10
|
||||
// sync_output_timestamps: true
|
||||
// update_frame_rate: false
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class PacketThinnerCalculator : public CalculatorBase {
|
||||
public:
|
||||
PacketThinnerCalculator() {}
|
||||
~PacketThinnerCalculator() override {}
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
|
||||
if (cc->InputSidePackets().HasTag(kPeriodTag)) {
|
||||
cc->InputSidePackets().Tag(kPeriodTag).Set<int64>();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (cc->InputTimestamp() < start_time_) {
|
||||
return ::mediapipe::OkStatus(); // Drop packets before start_time_.
|
||||
} else if (cc->InputTimestamp() >= end_time_) {
|
||||
if (!cc->Outputs().Index(0).IsClosed()) {
|
||||
cc->Outputs()
|
||||
.Index(0)
|
||||
.Close(); // No more Packets will be output after end_time_.
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
} else {
|
||||
return thinner_type_ == PacketThinnerCalculatorOptions::ASYNC
|
||||
? AsyncThinnerProcess(cc)
|
||||
: SyncThinnerProcess(cc);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
// Implementation of ASYNC and SYNC versions of thinner algorithm.
|
||||
::mediapipe::Status AsyncThinnerProcess(CalculatorContext* cc);
|
||||
::mediapipe::Status SyncThinnerProcess(CalculatorContext* cc);
|
||||
|
||||
// Cached option.
|
||||
PacketThinnerCalculatorOptions::ThinnerType thinner_type_;
|
||||
|
||||
// Given a Timestamp, finds the closest sync Timestamp
|
||||
// based on start_time_ and period_. This can be earlier or
|
||||
// later than given Timestamp, but is guaranteed to be within
|
||||
// half a period_.
|
||||
Timestamp NearestSyncTimestamp(Timestamp now) const;
|
||||
|
||||
// Cached option used by both async and sync thinners.
|
||||
TimestampDiff period_; // Interval during which only one packet is emitted.
|
||||
Timestamp start_time_; // Cached option - default Timestamp::Min()
|
||||
Timestamp end_time_; // Cached option - default Timestamp::Max()
|
||||
|
||||
// Only used by async thinner:
|
||||
Timestamp next_valid_timestamp_; // Suppress packets until this timestamp.
|
||||
|
||||
// Only used by sync thinner:
|
||||
Packet saved_packet_; // Best packet not yet emitted.
|
||||
bool sync_output_timestamps_; // Cached option.
|
||||
};
|
||||
REGISTER_CALCULATOR(PacketThinnerCalculator);
|
||||
|
||||
namespace {
|
||||
TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
|
||||
} // namespace
|
||||
|
||||
::mediapipe::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
|
||||
auto& options = cc->Options<PacketThinnerCalculatorOptions>();
|
||||
|
||||
thinner_type_ = options.thinner_type();
|
||||
// This check enables us to assume only two thinner types exist in Process()
|
||||
CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
|
||||
thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
|
||||
<< "Unsupported thinner type.";
|
||||
|
||||
if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
|
||||
// ASYNC thinner outputs packets with the same timestamp as their input so
|
||||
// its safe to SetOffset(0). SYNC thinner manipulates timestamps of its
|
||||
// output so we don't do this for that case.
|
||||
cc->SetOffset(0);
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag(kPeriodTag)) {
|
||||
period_ =
|
||||
TimestampDiff(cc->InputSidePackets().Tag(kPeriodTag).Get<int64>());
|
||||
} else {
|
||||
period_ = TimestampDiff(options.period());
|
||||
}
|
||||
CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive.";
|
||||
|
||||
if (options.has_start_time()) {
|
||||
start_time_ = Timestamp(options.start_time());
|
||||
} else if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
|
||||
start_time_ = Timestamp::Min();
|
||||
} else {
|
||||
start_time_ = Timestamp(0);
|
||||
}
|
||||
|
||||
end_time_ =
|
||||
options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
|
||||
CHECK_LT(start_time_, end_time_)
|
||||
<< "Invalid PacketThinner: start_time must be earlier than end_time";
|
||||
|
||||
sync_output_timestamps_ = options.sync_output_timestamps();
|
||||
|
||||
next_valid_timestamp_ = start_time_;
|
||||
// Drop packets until this time.
|
||||
cc->Outputs().Index(0).SetNextTimestampBound(start_time_);
|
||||
|
||||
if (!cc->Inputs().Index(0).Header().IsEmpty()) {
|
||||
if (options.update_frame_rate()) {
|
||||
const VideoHeader& video_header =
|
||||
cc->Inputs().Index(0).Header().Get<VideoHeader>();
|
||||
double new_frame_rate;
|
||||
if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
|
||||
new_frame_rate =
|
||||
video_header.frame_rate /
|
||||
ceil(video_header.frame_rate * options.period() / kTimebaseUs);
|
||||
} else {
|
||||
const double sampling_rate = kTimebaseUs / options.period();
|
||||
new_frame_rate = video_header.frame_rate < sampling_rate
|
||||
? video_header.frame_rate
|
||||
: sampling_rate;
|
||||
}
|
||||
std::unique_ptr<VideoHeader> header(new VideoHeader);
|
||||
header->format = video_header.format;
|
||||
header->width = video_header.width;
|
||||
header->height = video_header.height;
|
||||
header->frame_rate = new_frame_rate;
|
||||
cc->Outputs().Index(0).SetHeader(Adopt(header.release()));
|
||||
} else {
|
||||
cc->Outputs().Index(0).SetHeader(cc->Inputs().Index(0).Header());
|
||||
}
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
|
||||
// Emit any saved packets before quitting.
|
||||
if (!saved_packet_.IsEmpty()) {
|
||||
// Only sync thinner should have saved packets.
|
||||
CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
|
||||
if (sync_output_timestamps_) {
|
||||
cc->Outputs().Index(0).AddPacket(
|
||||
saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
|
||||
} else {
|
||||
cc->Outputs().Index(0).AddPacket(saved_packet_);
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status PacketThinnerCalculator::AsyncThinnerProcess(
|
||||
CalculatorContext* cc) {
|
||||
if (cc->InputTimestamp() >= next_valid_timestamp_) {
|
||||
cc->Outputs().Index(0).AddPacket(
|
||||
cc->Inputs().Index(0).Value()); // Emit current packet.
|
||||
next_valid_timestamp_ = cc->InputTimestamp() + period_;
|
||||
// Guaranteed not to emit packets seen during refractory period.
|
||||
cc->Outputs().Index(0).SetNextTimestampBound(next_valid_timestamp_);
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status PacketThinnerCalculator::SyncThinnerProcess(
|
||||
CalculatorContext* cc) {
|
||||
if (saved_packet_.IsEmpty()) {
|
||||
// If no packet has been saved, store the current packet.
|
||||
saved_packet_ = cc->Inputs().Index(0).Value();
|
||||
cc->Outputs().Index(0).SetNextTimestampBound(
|
||||
sync_output_timestamps_ ? NearestSyncTimestamp(cc->InputTimestamp())
|
||||
: cc->InputTimestamp());
|
||||
} else {
|
||||
// Saved packet exists -- update or emit.
|
||||
const Timestamp saved = saved_packet_.Timestamp();
|
||||
const Timestamp saved_sync = NearestSyncTimestamp(saved);
|
||||
const Timestamp now = cc->InputTimestamp();
|
||||
const Timestamp now_sync = NearestSyncTimestamp(now);
|
||||
CHECK_LE(saved_sync, now_sync);
|
||||
if (saved_sync == now_sync) {
|
||||
// Saved Packet is in same interval as current packet.
|
||||
// Replace saved packet with current if it is at least as
|
||||
// central as the saved packet wrt temporal interval.
|
||||
// [We break ties in favor of fresher packets]
|
||||
if (abs(now - now_sync) <= abs(saved - saved_sync)) {
|
||||
saved_packet_ = cc->Inputs().Index(0).Value();
|
||||
}
|
||||
} else {
|
||||
// Saved packet is the best packet from earlier interval: emit!
|
||||
if (sync_output_timestamps_) {
|
||||
cc->Outputs().Index(0).AddPacket(saved_packet_.At(saved_sync));
|
||||
cc->Outputs().Index(0).SetNextTimestampBound(now_sync);
|
||||
} else {
|
||||
cc->Outputs().Index(0).AddPacket(saved_packet_);
|
||||
cc->Outputs().Index(0).SetNextTimestampBound(now);
|
||||
}
|
||||
// Current packet is the first one we've seen from new interval -- save!
|
||||
saved_packet_ = cc->Inputs().Index(0).Value();
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
|
||||
CHECK_NE(start_time_, Timestamp::Unset())
|
||||
<< "Method only valid for sync thinner calculator.";
|
||||
|
||||
// Computation is done using int64 arithmetic. No easy way to avoid
|
||||
// since Timestamps don't support div and multiply.
|
||||
const int64 now64 = now.Value();
|
||||
const int64 start64 = start_time_.Value();
|
||||
const int64 period64 = period_.Value();
|
||||
CHECK_LE(0, period64);
|
||||
|
||||
// Round now64 to its closest interval (units of period64).
|
||||
int64 sync64 =
|
||||
(now64 - start64 + period64 / 2) / period64 * period64 + start64;
|
||||
CHECK_LE(abs(now64 - sync64), period64 / 2)
|
||||
<< "start64: " << start64 << "; now64: " << now64
|
||||
<< "; sync64: " << sync64;
|
||||
|
||||
return Timestamp(sync64);
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2018 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message PacketThinnerCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional PacketThinnerCalculatorOptions ext = 288533508;
|
||||
}
|
||||
|
||||
enum ThinnerType {
|
||||
ASYNC = 1; // Asynchronous thinner, described below [default].
|
||||
SYNC = 2; // Synchronous thinner, also described below.
|
||||
}
|
||||
optional ThinnerType thinner_type = 1 [default = ASYNC];
|
||||
|
||||
// The period (in microsecond) specifies the temporal interval during which
|
||||
// only a single packet is emitted in the output stream. Has subtly different
|
||||
// semantics depending on the thinner type, as follows.
|
||||
//
|
||||
// Async thinner: this option is a refractory period -- once a packet is
|
||||
// emitted, we guarantee that no packets will be emitted for period ticks.
|
||||
//
|
||||
// Sync thinner: the period specifies a temporal interval during which
|
||||
// only one packet is emitted. The emitted packet is guaranteed to be
|
||||
// the one closest to the center of the temporal interval (no guarantee on
|
||||
// how ties are broken). More specifically,
|
||||
// intervals are centered at start_time + i * period
|
||||
// (for non-negative integers i).
|
||||
// Thus, each interval extends period/2 ticks before and after its center.
|
||||
// Additionally, in the sync thinner any packets earlier than start_time
|
||||
// are discarded and the thinner calls Close() once timestamp equals or
|
||||
// exceeds end_time.
|
||||
optional int64 period = 2 [default = 1];
|
||||
|
||||
// Packets before start_time and at/after end_time are discarded.
|
||||
// Additionally, for a sync thinner, start time specifies the center of
|
||||
// time invervals as described above and therefore should be set explicitly.
|
||||
optional int64 start_time = 3; // If not specified, set to 0 for SYNC type,
|
||||
// and set to Timestamp::Min() for ASYNC type.
|
||||
optional int64 end_time = 4; // Set to Timestamp::Max() if not specified.
|
||||
|
||||
// Whether the timestamps of packets emitted by sync thinner should
|
||||
// correspond to the center of their corresponding temporal interval.
|
||||
// If false, packets emitted using original timestamp (as in async thinner).
|
||||
optional bool sync_output_timestamps = 5 [default = true];
|
||||
|
||||
// If true, update the frame rate in the header, if it's available, to an
|
||||
// estimated frame rate due to the sampling.
|
||||
optional bool update_frame_rate = 6 [default = false];
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/video_stream_header.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
// A simple version of CalculatorRunner with built-in convenience methods for
|
||||
// setting inputs from a vector and checking outputs against a vector of
|
||||
// expected outputs.
|
||||
class SimpleRunner : public CalculatorRunner {
|
||||
public:
|
||||
explicit SimpleRunner(const CalculatorOptions& options)
|
||||
: CalculatorRunner("PacketThinnerCalculator", options) {
|
||||
SetNumInputs(1);
|
||||
SetNumOutputs(1);
|
||||
SetNumInputSidePackets(0);
|
||||
}
|
||||
|
||||
explicit SimpleRunner(const CalculatorGraphConfig::Node& node)
|
||||
: CalculatorRunner(node) {}
|
||||
|
||||
void SetInput(const std::vector<int>& timestamp_list) {
|
||||
MutableInputs()->Index(0).packets.clear();
|
||||
for (const int ts : timestamp_list) {
|
||||
MutableInputs()->Index(0).packets.push_back(
|
||||
MakePacket<std::string>(absl::StrCat("Frame #", ts))
|
||||
.At(Timestamp(ts)));
|
||||
}
|
||||
}
|
||||
|
||||
void SetFrameRate(const double frame_rate) {
|
||||
auto video_header = absl::make_unique<VideoHeader>();
|
||||
video_header->frame_rate = frame_rate;
|
||||
MutableInputs()->Index(0).header = Adopt(video_header.release());
|
||||
}
|
||||
|
||||
std::vector<int64> GetOutputTimestamps() const {
|
||||
std::vector<int64> timestamps;
|
||||
for (const Packet& packet : Outputs().Index(0).packets) {
|
||||
timestamps.emplace_back(packet.Timestamp().Value());
|
||||
}
|
||||
return timestamps;
|
||||
}
|
||||
|
||||
double GetFrameRate() const {
|
||||
CHECK(!Outputs().Index(0).header.IsEmpty());
|
||||
return Outputs().Index(0).header.Get<VideoHeader>().frame_rate;
|
||||
}
|
||||
};
|
||||
|
||||
// Check that thinner respects start_time and end_time options.
|
||||
// We only test with one thinner because the logic for start & end time
|
||||
// handling is shared across both types of thinner in Process().
|
||||
TEST(PacketThinnerCalculatorTest, StartAndEndTimeTest) {
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
|
||||
extension->set_period(5);
|
||||
extension->set_start_time(4);
|
||||
extension->set_end_time(12);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {5, 11};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, AsyncUniformStreamThinningTest) {
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
|
||||
extension->set_period(5);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 8, 14};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, ASyncUniformStreamThinningTestBySidePacket) {
|
||||
// Note: sync runner but outputting *original* timestamps.
|
||||
CalculatorGraphConfig::Node node;
|
||||
node.set_calculator("PacketThinnerCalculator");
|
||||
node.add_input_side_packet("PERIOD:period");
|
||||
node.add_input_stream("input_stream");
|
||||
node.add_output_stream("output_stream");
|
||||
auto* extension = node.mutable_options()->MutableExtension(
|
||||
PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
|
||||
extension->set_start_time(0);
|
||||
extension->set_sync_output_timestamps(false);
|
||||
|
||||
SimpleRunner runner(node);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 8, 14};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest1) {
|
||||
// Note: sync runner but outputting *original* timestamps.
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
|
||||
extension->set_start_time(0);
|
||||
extension->set_period(5);
|
||||
extension->set_sync_output_timestamps(false);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTestBySidePacket1) {
|
||||
// Note: sync runner but outputting *original* timestamps.
|
||||
CalculatorGraphConfig::Node node;
|
||||
node.set_calculator("PacketThinnerCalculator");
|
||||
node.add_input_side_packet("PERIOD:period");
|
||||
node.add_input_stream("input_stream");
|
||||
node.add_output_stream("output_stream");
|
||||
auto* extension = node.mutable_options()->MutableExtension(
|
||||
PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
|
||||
extension->set_start_time(0);
|
||||
extension->set_sync_output_timestamps(false);
|
||||
|
||||
SimpleRunner runner(node);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest2) {
|
||||
// Same test but now with synced timestamps.
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
|
||||
extension->set_start_time(0);
|
||||
extension->set_period(5);
|
||||
extension->set_sync_output_timestamps(true);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {0, 5, 10, 15};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
// Test: Given a stream with timestamps corresponding to first ten prime numbers
|
||||
// and period of 5, confirm whether timestamps of thinner stream matches
|
||||
// expectations.
|
||||
TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest1) {
|
||||
// ASYNC thinner.
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
|
||||
extension->set_period(5);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 7, 13, 19, 29};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest2) {
|
||||
// SYNC with original timestamps.
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
|
||||
extension->set_start_time(0);
|
||||
extension->set_period(5);
|
||||
extension->set_sync_output_timestamps(false);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 5, 11, 17, 19, 23, 29};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
// Confirm that Calculator correctly handles boundary cases.
|
||||
TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest1) {
|
||||
// Odd period, negative start_time
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
|
||||
extension->set_start_time(-10);
|
||||
extension->set_period(5);
|
||||
extension->set_sync_output_timestamps(true);
|
||||
SimpleRunner runner(options);
|
||||
// Two timestamps falling on either side of a period boundary.
|
||||
runner.SetInput({2, 3});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {0, 5};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest2) {
|
||||
// Even period, negative start_time, negative packet timestamps.
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
|
||||
extension->set_start_time(-144);
|
||||
extension->set_period(6);
|
||||
extension->set_sync_output_timestamps(true);
|
||||
SimpleRunner runner(options);
|
||||
// Two timestamps falling on either side of a period boundary.
|
||||
runner.SetInput({-4, -3, 8, 9});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {-6, 0, 6, 12};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, FrameRateTest1) {
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
|
||||
extension->set_period(5);
|
||||
extension->set_update_frame_rate(true);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
runner.SetFrameRate(1000000.0 / 2);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 8, 14};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
// The true sampling period is 6.
|
||||
EXPECT_DOUBLE_EQ(1000000.0 / 6, runner.GetFrameRate());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, FrameRateTest2) {
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
|
||||
extension->set_period(5);
|
||||
extension->set_update_frame_rate(true);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({8, 16, 24, 32, 40, 48, 56});
|
||||
runner.SetFrameRate(1000000.0 / 8);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<int64> expected_timestamps = {8, 16, 24, 32, 40, 48, 56};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
// The true sampling period is still 8.
|
||||
EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, FrameRateTest3) {
|
||||
// Note: sync runner but outputting *original* timestamps.
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
|
||||
extension->set_start_time(0);
|
||||
extension->set_period(5);
|
||||
extension->set_sync_output_timestamps(false);
|
||||
extension->set_update_frame_rate(true);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
runner.SetFrameRate(1000000.0 / 2);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
// The true (long-run) sampling period is 5.
|
||||
EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, FrameRateTest4) {
|
||||
// Same test but now with synced timestamps.
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
|
||||
extension->set_start_time(0);
|
||||
extension->set_period(5);
|
||||
extension->set_sync_output_timestamps(true);
|
||||
extension->set_update_frame_rate(true);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
|
||||
runner.SetFrameRate(1000000.0 / 2);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {0, 5, 10, 15};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
// The true (long-run) sampling period is 5.
|
||||
EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate());
|
||||
}
|
||||
|
||||
TEST(PacketThinnerCalculatorTest, FrameRateTest5) {
|
||||
CalculatorOptions options;
|
||||
auto* extension =
|
||||
options.MutableExtension(PacketThinnerCalculatorOptions::ext);
|
||||
extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
|
||||
extension->set_start_time(0);
|
||||
extension->set_period(5);
|
||||
extension->set_sync_output_timestamps(true);
|
||||
extension->set_update_frame_rate(true);
|
||||
SimpleRunner runner(options);
|
||||
runner.SetInput({8, 16, 24, 32, 40, 48, 56});
|
||||
runner.SetFrameRate(1000000.0 / 8);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<int64> expected_timestamps = {10, 15, 25, 30, 40, 50, 55};
|
||||
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
|
||||
// The true (long-run) sampling period is 8.
|
||||
EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -17,6 +17,7 @@
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
@@ -86,6 +87,7 @@ class PreviousLoopbackCalculator : public CalculatorBase {
|
||||
main_ts_.pop_front();
|
||||
}
|
||||
}
|
||||
auto& loop_out = cc->Outputs().Get(loop_out_id_);
|
||||
|
||||
while (!main_ts_.empty() && !loopback_packets_.empty()) {
|
||||
Timestamp main_timestamp = main_ts_.front();
|
||||
@@ -95,13 +97,32 @@ class PreviousLoopbackCalculator : public CalculatorBase {
|
||||
|
||||
if (previous_loopback.IsEmpty()) {
|
||||
// TODO: SetCompleteTimestampBound would be more useful.
|
||||
cc->Outputs()
|
||||
.Get(loop_out_id_)
|
||||
.SetNextTimestampBound(main_timestamp + 1);
|
||||
loop_out.SetNextTimestampBound(main_timestamp + 1);
|
||||
} else {
|
||||
cc->Outputs().Get(loop_out_id_).AddPacket(std::move(previous_loopback));
|
||||
loop_out.AddPacket(std::move(previous_loopback));
|
||||
}
|
||||
}
|
||||
|
||||
// In case of an empty loopback input, the next timestamp bound for
|
||||
// loopback input is the loopback timestamp + 1. The next timestamp bound
|
||||
// for output is set and the main_ts_ vector is truncated accordingly.
|
||||
if (loopback_packet.IsEmpty() &&
|
||||
loopback_packet.Timestamp() != Timestamp::Unstarted()) {
|
||||
Timestamp loopback_bound =
|
||||
loopback_packet.Timestamp().NextAllowedInStream();
|
||||
while (!main_ts_.empty() && main_ts_.front() <= loopback_bound) {
|
||||
main_ts_.pop_front();
|
||||
}
|
||||
if (main_ts_.empty()) {
|
||||
loop_out.SetNextTimestampBound(loopback_bound.NextAllowedInStream());
|
||||
}
|
||||
}
|
||||
if (!main_ts_.empty()) {
|
||||
loop_out.SetNextTimestampBound(main_ts_.front());
|
||||
}
|
||||
if (cc->Inputs().Get(main_id_).IsDone() && main_ts_.empty()) {
|
||||
loop_out.Close();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
||||
@@ -74,11 +74,11 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
tool::AddVectorSink("pair", &graph_config_, &in_prev);
|
||||
|
||||
CalculatorGraph graph_;
|
||||
MEDIAPIPE_ASSERT_OK(graph_.Initialize(graph_config_, {}));
|
||||
MEDIAPIPE_ASSERT_OK(graph_.StartRun({}));
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
|
||||
MP_ASSERT_OK(graph_.StartRun({}));
|
||||
|
||||
auto send_packet = [&graph_](const std::string& input_name, int n) {
|
||||
MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
MP_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
input_name, MakePacket<int>(n).At(Timestamp(n))));
|
||||
};
|
||||
auto pair_values = [](const Packet& packet) {
|
||||
@@ -89,22 +89,181 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
};
|
||||
|
||||
send_packet("in", 1);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1}));
|
||||
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(1, -1));
|
||||
|
||||
send_packet("in", 2);
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2}));
|
||||
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(2, 1));
|
||||
|
||||
send_packet("in", 5);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5}));
|
||||
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 1));
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2, 5}));
|
||||
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 2));
|
||||
|
||||
send_packet("in", 15);
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5, 15}));
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2, 5, 15}));
|
||||
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(15, 5));
|
||||
|
||||
MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MEDIAPIPE_EXPECT_OK(graph_.WaitUntilDone());
|
||||
MP_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MP_EXPECT_OK(graph_.WaitUntilDone());
|
||||
}
|
||||
|
||||
// A Calculator that outputs a summary packet in CalculatorBase::Close().
|
||||
class PacketOnCloseCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<int>();
|
||||
cc->Outputs().Index(0).Set<int>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) final {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) final {
|
||||
sum_ += cc->Inputs().Index(0).Value().Get<int>();
|
||||
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Close(CalculatorContext* cc) final {
|
||||
cc->Outputs().Index(0).AddPacket(
|
||||
MakePacket<int>(sum_).At(Timestamp::Max()));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
int sum_ = 0;
|
||||
};
|
||||
REGISTER_CALCULATOR(PacketOnCloseCalculator);
|
||||
|
||||
// Demonstrates that all ouput and input streams in PreviousLoopbackCalculator
|
||||
// will close as expected when all graph input streams are closed.
|
||||
TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
|
||||
std::vector<Packet> outputs;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
input_stream: 'MAIN:in'
|
||||
input_stream: 'LOOP:out'
|
||||
input_stream_info: { tag_index: 'LOOP' back_edge: true }
|
||||
output_stream: 'PREV_LOOP:previous'
|
||||
}
|
||||
# This calculator synchronizes its inputs as normal, so it is used
|
||||
# to check that both "in" and "previous" are ready.
|
||||
node {
|
||||
calculator: 'PassThroughCalculator'
|
||||
input_stream: 'in'
|
||||
input_stream: 'previous'
|
||||
output_stream: 'out'
|
||||
output_stream: 'previous2'
|
||||
}
|
||||
node {
|
||||
calculator: 'PacketOnCloseCalculator'
|
||||
input_stream: 'out'
|
||||
output_stream: 'close_out'
|
||||
}
|
||||
)");
|
||||
tool::AddVectorSink("close_out", &graph_config_, &outputs);
|
||||
|
||||
CalculatorGraph graph_;
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
|
||||
MP_ASSERT_OK(graph_.StartRun({}));
|
||||
|
||||
auto send_packet = [&graph_](const std::string& input_name, int n) {
|
||||
MP_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
input_name, MakePacket<int>(n).At(Timestamp(n))));
|
||||
};
|
||||
|
||||
send_packet("in", 1);
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1}));
|
||||
|
||||
send_packet("in", 2);
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2}));
|
||||
|
||||
send_packet("in", 5);
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2, 5}));
|
||||
|
||||
send_packet("in", 15);
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2, 5, 15}));
|
||||
|
||||
MP_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(outputs),
|
||||
(std::vector<int64>{1, 2, 5, 15, Timestamp::Max().Value()}));
|
||||
|
||||
MP_EXPECT_OK(graph_.WaitUntilDone());
|
||||
}
|
||||
|
||||
// Demonstrates that downstream calculators won't be blocked by
|
||||
// always-empty-LOOP-stream.
|
||||
TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
|
||||
std::vector<Packet> outputs;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
input_stream: 'MAIN:in'
|
||||
input_stream: 'LOOP:previous'
|
||||
input_stream_info: { tag_index: 'LOOP' back_edge: true }
|
||||
output_stream: 'PREV_LOOP:previous'
|
||||
}
|
||||
# This calculator synchronizes its inputs as normal, so it is used
|
||||
# to check that both "in" and "previous" are ready.
|
||||
node {
|
||||
calculator: 'PassThroughCalculator'
|
||||
input_stream: 'in'
|
||||
input_stream: 'previous'
|
||||
output_stream: 'out'
|
||||
output_stream: 'previous2'
|
||||
}
|
||||
node {
|
||||
calculator: 'PacketOnCloseCalculator'
|
||||
input_stream: 'out'
|
||||
output_stream: 'close_out'
|
||||
}
|
||||
)");
|
||||
tool::AddVectorSink("close_out", &graph_config_, &outputs);
|
||||
|
||||
CalculatorGraph graph_;
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
|
||||
MP_ASSERT_OK(graph_.StartRun({}));
|
||||
|
||||
auto send_packet = [&graph_](const std::string& input_name, int n) {
|
||||
MP_EXPECT_OK(graph_.AddPacketToInputStream(
|
||||
input_name, MakePacket<int>(n).At(Timestamp(n))));
|
||||
};
|
||||
|
||||
send_packet("in", 0);
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{0}));
|
||||
|
||||
for (int main_ts = 1; main_ts < 50; ++main_ts) {
|
||||
send_packet("in", main_ts);
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
std::vector<int64> ts_values = TimestampValues(outputs);
|
||||
EXPECT_EQ(ts_values.size(), main_ts + 1);
|
||||
for (int j = 0; j < main_ts; ++j) {
|
||||
EXPECT_EQ(ts_values[j], j);
|
||||
}
|
||||
}
|
||||
|
||||
MP_EXPECT_OK(graph_.CloseAllInputStreams());
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
MP_EXPECT_OK(graph_.WaitUntilDone());
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cfloat>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/core/quantize_float_vector_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
// Quantizes a vector of floats to a std::string so that each float becomes a
|
||||
// byte in the [0, 255] range. Any value above max_quantized_value or below
|
||||
// min_quantized_value will be saturated to '/xFF' or '/0'.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "QuantizeFloatVectorCalculator"
|
||||
// input_stream: "FLOAT_VECTOR:float_vector"
|
||||
// output_stream: "ENCODED:encoded"
|
||||
// options {
|
||||
// [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
|
||||
// max_quantized_value: 64
|
||||
// min_quantized_value: -64
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
namespace mediapipe {
|
||||
|
||||
class QuantizeFloatVectorCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
|
||||
cc->Outputs().Tag("ENCODED").Set<std::string>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) final {
|
||||
const auto options =
|
||||
cc->Options<::mediapipe::QuantizeFloatVectorCalculatorOptions>();
|
||||
if (!options.has_max_quantized_value() ||
|
||||
!options.has_min_quantized_value()) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"Both max_quantized_value and min_quantized_value must be provided "
|
||||
"in QuantizeFloatVectorCalculatorOptions.");
|
||||
}
|
||||
max_quantized_value_ = options.max_quantized_value();
|
||||
min_quantized_value_ = options.min_quantized_value();
|
||||
if (max_quantized_value_ < min_quantized_value_ + FLT_EPSILON) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"max_quantized_value must be greater than min_quantized_value.");
|
||||
}
|
||||
range_ = max_quantized_value_ - min_quantized_value_;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) final {
|
||||
const std::vector<float>& float_vector =
|
||||
cc->Inputs().Tag("FLOAT_VECTOR").Value().Get<std::vector<float>>();
|
||||
int feature_size = float_vector.size();
|
||||
std::string encoded_features;
|
||||
encoded_features.reserve(feature_size);
|
||||
for (int i = 0; i < feature_size; i++) {
|
||||
float old_value = float_vector[i];
|
||||
if (old_value < min_quantized_value_) {
|
||||
old_value = min_quantized_value_;
|
||||
}
|
||||
if (old_value > max_quantized_value_) {
|
||||
old_value = max_quantized_value_;
|
||||
}
|
||||
unsigned char encoded = static_cast<unsigned char>(
|
||||
(old_value - min_quantized_value_) * (255.0 / range_));
|
||||
encoded_features += encoded;
|
||||
}
|
||||
cc->Outputs().Tag("ENCODED").AddPacket(
|
||||
MakePacket<std::string>(encoded_features).At(cc->InputTimestamp()));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
float max_quantized_value_;
|
||||
float min_quantized_value_;
|
||||
float range_;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(QuantizeFloatVectorCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message QuantizeFloatVectorCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional QuantizeFloatVectorCalculatorOptions ext = 259848061;
|
||||
}
|
||||
|
||||
optional float max_quantized_value = 1;
|
||||
optional float min_quantized_value = 2;
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
options {
|
||||
[mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
testing::HasSubstr(
|
||||
"Both max_quantized_value and min_quantized_value must be provided"));
|
||||
}
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
options {
|
||||
[mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
|
||||
max_quantized_value: -1
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
testing::HasSubstr(
|
||||
"max_quantized_value must be greater than min_quantized_value"));
|
||||
}
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
options {
|
||||
[mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
|
||||
max_quantized_value: 1
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
|
||||
auto status = runner.Run();
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
testing::HasSubstr(
|
||||
"max_quantized_value must be greater than min_quantized_value"));
|
||||
}
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
options {
|
||||
[mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
|
||||
max_quantized_value: 1
|
||||
min_quantized_value: -1
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_TRUE(outputs[0].Get<std::string>().empty());
|
||||
EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
|
||||
}
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
options {
|
||||
[mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
|
||||
max_quantized_value: 64
|
||||
min_quantized_value: -64
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f};
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
const std::string& result = outputs[0].Get<std::string>();
|
||||
ASSERT_FALSE(result.empty());
|
||||
EXPECT_EQ(5, result.size());
|
||||
// 127
|
||||
EXPECT_EQ('\x7F', result.c_str()[0]);
|
||||
// 0
|
||||
EXPECT_EQ('\0', result.c_str()[1]);
|
||||
// 255
|
||||
EXPECT_EQ('\xFF', result.c_str()[2]);
|
||||
// 63
|
||||
EXPECT_EQ('\x3F', result.c_str()[3]);
|
||||
// 191
|
||||
EXPECT_EQ('\xBF', result.c_str()[4]);
|
||||
EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
|
||||
}
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
options {
|
||||
[mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
|
||||
max_quantized_value: 64
|
||||
min_quantized_value: -64
|
||||
}
|
||||
}
|
||||
)");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> vector = {-65.0f, 65.0f};
|
||||
runner.MutableInputs()
|
||||
->Tag("FLOAT_VECTOR")
|
||||
.packets.push_back(
|
||||
MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
const std::string& result = outputs[0].Get<std::string>();
|
||||
ASSERT_FALSE(result.empty());
|
||||
EXPECT_EQ(2, result.size());
|
||||
// 0
|
||||
EXPECT_EQ('\0', result.c_str()[0]);
|
||||
// 255
|
||||
EXPECT_EQ('\xFF', result.c_str()[1]);
|
||||
EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,114 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <deque>
|
||||
|
||||
#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// A Calculator that shifts the timestamps of packets along a stream. Packets on
|
||||
// the input stream are output with a timestamp of the packet given by packet
|
||||
// offset. That is, packet[i] is output with the timestamp of
|
||||
// packet[i + packet_offset]. Packet offset can be either positive or negative.
|
||||
// If packet_offset is -n, the first n packets will be dropped. If packet offset
|
||||
// is n, the final n packets will be dropped. For example, with a packet_offset
|
||||
// of -1, the first packet on the stream will be dropped, the second will be
|
||||
// output with the timestamp of the first, the third with the timestamp of the
|
||||
// second, and so on.
|
||||
class SequenceShiftCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
// Reads from options to set cache_size_ and packet_offset_.
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// A positive offset means we want a packet to be output with the timestamp of
|
||||
// a later packet. Stores packets waiting for their output timestamps and
|
||||
// outputs a single packet when the cache fills.
|
||||
void ProcessPositiveOffset(CalculatorContext* cc);
|
||||
|
||||
// A negative offset means we want a packet to be output with the timestamp of
|
||||
// an earlier packet. Stores timestamps waiting for the corresponding input
|
||||
// packet and outputs a single packet when the cache fills.
|
||||
void ProcessNegativeOffset(CalculatorContext* cc);
|
||||
|
||||
// Storage for packets waiting to be output when packet_offset > 0. When cache
|
||||
// is full, oldest packet is output with current timestamp.
|
||||
std::deque<Packet> packet_cache_;
|
||||
|
||||
// Storage for previous timestamps used when packet_offset < 0. When cache is
|
||||
// full, oldest timestamp is used for current packet.
|
||||
std::deque<Timestamp> timestamp_cache_;
|
||||
|
||||
// Copied from corresponding field in options.
|
||||
int packet_offset_;
|
||||
// The number of packets or timestamps we need to store to output packet[i] at
|
||||
// the timestamp of packet[i + packet_offset]; equal to abs(packet_offset).
|
||||
int cache_size_;
|
||||
};
|
||||
REGISTER_CALCULATOR(SequenceShiftCalculator);
|
||||
|
||||
::mediapipe::Status SequenceShiftCalculator::Open(CalculatorContext* cc) {
|
||||
packet_offset_ =
|
||||
cc->Options<mediapipe::SequenceShiftCalculatorOptions>().packet_offset();
|
||||
cache_size_ = abs(packet_offset_);
|
||||
// An offset of zero is a no-op, but someone might still request it.
|
||||
if (packet_offset_ == 0) {
|
||||
cc->Outputs().Index(0).SetOffset(0);
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status SequenceShiftCalculator::Process(CalculatorContext* cc) {
|
||||
if (packet_offset_ > 0) {
|
||||
ProcessPositiveOffset(cc);
|
||||
} else if (packet_offset_ < 0) {
|
||||
ProcessNegativeOffset(cc);
|
||||
} else {
|
||||
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
|
||||
if (packet_cache_.size() >= cache_size_) {
|
||||
// Ready to output oldest packet with current timestamp.
|
||||
cc->Outputs().Index(0).AddPacket(
|
||||
packet_cache_.front().At(cc->InputTimestamp()));
|
||||
packet_cache_.pop_front();
|
||||
}
|
||||
// Store current packet for later output.
|
||||
packet_cache_.push_back(cc->Inputs().Index(0).Value());
|
||||
}
|
||||
|
||||
void SequenceShiftCalculator::ProcessNegativeOffset(CalculatorContext* cc) {
|
||||
if (timestamp_cache_.size() >= cache_size_) {
|
||||
// Ready to output current packet with oldest timestamp.
|
||||
cc->Outputs().Index(0).AddPacket(
|
||||
cc->Inputs().Index(0).Value().At(timestamp_cache_.front()));
|
||||
timestamp_cache_.pop_front();
|
||||
}
|
||||
// Store current timestamp for use by a future packet.
|
||||
timestamp_cache_.push_back(cc->InputTimestamp());
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message SequenceShiftCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional SequenceShiftCalculatorOptions ext = 107633927;
|
||||
}
|
||||
optional int32 packet_offset = 1 [default = -1];
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
// Adds packets containing integers equal to their original timestamp.
|
||||
void AddPackets(CalculatorRunner* runner) {
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
runner->MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(new int(i)).At(Timestamp(i)));
|
||||
}
|
||||
}
|
||||
|
||||
// Zero shift is a no-op (output input[i] at timestamp[i]). Input and output
|
||||
// streams should be identical.
|
||||
TEST(SequenceShiftCalculatorTest, ZeroShift) {
|
||||
CalculatorRunner runner(
|
||||
"SequenceShiftCalculator",
|
||||
"[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: 0 }", 1,
|
||||
1, 0);
|
||||
AddPackets(&runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& input_packets =
|
||||
runner.MutableInputs()->Index(0).packets;
|
||||
const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
|
||||
ASSERT_EQ(10, input_packets.size());
|
||||
ASSERT_EQ(input_packets.size(), output_packets.size());
|
||||
for (int i = 0; i < output_packets.size(); ++i) {
|
||||
// Make sure the contents are as expected.
|
||||
EXPECT_EQ(input_packets[i].Get<int>(), output_packets[i].Get<int>());
|
||||
EXPECT_EQ(input_packets[i].Timestamp(), output_packets[i].Timestamp());
|
||||
}
|
||||
}
|
||||
|
||||
// Tests shifting by three packets, i.e., output input[i] with the timestamp of
|
||||
// input[i + 3].
|
||||
TEST(SequenceShiftCalculatorTest, PositiveShift) {
|
||||
CalculatorRunner runner(
|
||||
"SequenceShiftCalculator",
|
||||
"[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: 3 }", 1,
|
||||
1, 0);
|
||||
AddPackets(&runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& input_packets =
|
||||
runner.MutableInputs()->Index(0).packets;
|
||||
const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
|
||||
ASSERT_EQ(10, input_packets.size());
|
||||
// input_packet[i] should be output with the timestamp of input_packet[i + 3].
|
||||
// The last 3 packets are dropped.
|
||||
ASSERT_EQ(7, output_packets.size());
|
||||
for (int i = 0; i < output_packets.size(); ++i) {
|
||||
// Make sure the contents are as expected.
|
||||
EXPECT_EQ(input_packets[i].Get<int>(), output_packets[i].Get<int>());
|
||||
// Make sure the timestamps are shifted as expected.
|
||||
EXPECT_EQ(input_packets[i + 3].Timestamp(), output_packets[i].Timestamp());
|
||||
}
|
||||
}
|
||||
|
||||
// Tests shifting by -2, i.e., output input[i] with timestamp[i - 2]. The first
|
||||
// two packets should be dropped.
|
||||
TEST(SequenceShiftCalculatorTest, NegativeShift) {
|
||||
CalculatorRunner runner(
|
||||
"SequenceShiftCalculator",
|
||||
"[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: -2 }",
|
||||
1, 1, 0);
|
||||
AddPackets(&runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const std::vector<Packet>& input_packets =
|
||||
runner.MutableInputs()->Index(0).packets;
|
||||
const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
|
||||
ASSERT_EQ(10, input_packets.size());
|
||||
// Input packet[i] should be output with the timestamp of input packet[i - 2].
|
||||
// The first two packets are dropped. This means timestamps match between
|
||||
// input and output packets, but the data in the output packets come from
|
||||
// input_packets[i + 2].
|
||||
ASSERT_EQ(8, output_packets.size());
|
||||
for (int i = 0; i < output_packets.size(); ++i) {
|
||||
EXPECT_EQ(input_packets[i].Timestamp(), output_packets[i].Timestamp());
|
||||
EXPECT_EQ(input_packets[i + 2].Get<int>(), output_packets[i].Get<int>());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
using mediapipe::PacketTypeSet;
|
||||
using mediapipe::Timestamp;
|
||||
|
||||
namespace {
|
||||
static std::map<std::string, Timestamp>* kTimestampMap = []() {
|
||||
auto* res = new std::map<std::string, Timestamp>();
|
||||
res->emplace("AT_PRESTREAM", Timestamp::PreStream());
|
||||
res->emplace("AT_POSTSTREAM", Timestamp::PostStream());
|
||||
res->emplace("AT_ZERO", Timestamp(0));
|
||||
return res;
|
||||
}();
|
||||
|
||||
} // namespace
|
||||
|
||||
// Outputs the single input_side_packet at the timestamp specified in the
|
||||
// output_stream tag. Valid tags are AT_PRESTREAM, AT_POSTSTREAM and AT_ZERO.
|
||||
class SidePacketToStreamCalculator : public CalculatorBase {
|
||||
public:
|
||||
SidePacketToStreamCalculator() = default;
|
||||
~SidePacketToStreamCalculator() override = default;
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
|
||||
|
||||
::mediapipe::Status SidePacketToStreamCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
cc->InputSidePackets().Index(0).SetAny();
|
||||
|
||||
std::set<std::string> tags = cc->Outputs().GetTags();
|
||||
RET_CHECK_EQ(tags.size(), 1);
|
||||
|
||||
RET_CHECK_EQ(kTimestampMap->count(*tags.begin()), 1);
|
||||
cc->Outputs().Tag(*tags.begin()).SetAny();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status SidePacketToStreamCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
return mediapipe::tool::StatusStop();
|
||||
}
|
||||
|
||||
::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
|
||||
std::set<std::string> tags = cc->Outputs().GetTags();
|
||||
RET_CHECK_EQ(tags.size(), 1);
|
||||
const std::string& tag = *tags.begin();
|
||||
RET_CHECK_EQ(kTimestampMap->count(tag), 1);
|
||||
cc->Outputs().Tag(tag).AddPacket(
|
||||
cc->InputSidePackets().Index(0).At(kTimestampMap->at(tag)));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/split_vector_calculator.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
// input_stream: "tflitetensor_vector"
|
||||
// output_stream: "tflitetensor_vector_range_0"
|
||||
// output_stream: "tflitetensor_vector_range_1"
|
||||
// options {
|
||||
// [mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
// ranges: { begin: 0 end: 1 }
|
||||
// ranges: { begin: 1 end: 2 }
|
||||
// element_only: false
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
typedef SplitVectorCalculator<TfLiteTensor, false>
|
||||
SplitTfLiteTensorVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark, false>
|
||||
SplitLandmarkVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<::mediapipe::NormalizedRect, false>
|
||||
SplitNormalizedRectVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true>
|
||||
MovableSplitGlBufferVectorCalculator;
|
||||
REGISTER_CALCULATOR(MovableSplitGlBufferVectorCalculator);
|
||||
#endif
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,254 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_
|
||||
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
#include "tensorflow/lite/error_reporter.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
#include "tensorflow/lite/kernels/register.h"
|
||||
#include "tensorflow/lite/model.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
template <typename T>
|
||||
using IsCopyable = std::enable_if_t<std::is_copy_constructible<T>::value, bool>;
|
||||
|
||||
template <typename T>
|
||||
using IsNotCopyable =
|
||||
std::enable_if_t<!std::is_copy_constructible<T>::value, bool>;
|
||||
|
||||
template <typename T>
|
||||
using IsMovable = std::enable_if_t<std::is_move_constructible<T>::value, bool>;
|
||||
|
||||
template <typename T>
|
||||
using IsNotMovable =
|
||||
std::enable_if_t<!std::is_move_constructible<T>::value, bool>;
|
||||
|
||||
// Splits an input packet with std::vector<T> into multiple std::vector<T>
|
||||
// output packets using the [begin, end) ranges specified in
|
||||
// SplitVectorCalculatorOptions. If the option "element_only" is set to true,
|
||||
// all ranges should be of size 1 and all outputs will be elements of type T. If
|
||||
// "element_only" is false, ranges can be non-zero in size and all outputs will
|
||||
// be of type std::vector<T>. If the option "combine_outputs" is set to true,
|
||||
// only one output stream can be specified and all ranges of elements will be
|
||||
// combined into one vector.
|
||||
// To use this class for a particular type T, register a calculator using
|
||||
// SplitVectorCalculator<T>.
|
||||
template <typename T, bool move_elements>
|
||||
class SplitVectorCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().NumEntries() == 1);
|
||||
RET_CHECK(cc->Outputs().NumEntries() != 0);
|
||||
|
||||
cc->Inputs().Index(0).Set<std::vector<T>>();
|
||||
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
|
||||
|
||||
if (!std::is_copy_constructible<T>::value || move_elements) {
|
||||
// Ranges of elements shouldn't overlap when the vector contains
|
||||
// non-copyable elements.
|
||||
RET_CHECK_OK(checkRangesDontOverlap(options));
|
||||
}
|
||||
|
||||
if (options.combine_outputs()) {
|
||||
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
|
||||
cc->Outputs().Index(0).Set<std::vector<T>>();
|
||||
RET_CHECK_OK(checkRangesDontOverlap(options));
|
||||
} else {
|
||||
if (cc->Outputs().NumEntries() != options.ranges_size()) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"The number of output streams should match the number of ranges "
|
||||
"specified in the CalculatorOptions.");
|
||||
}
|
||||
|
||||
// Set the output types for each output stream.
|
||||
for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
|
||||
if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 ||
|
||||
options.ranges(i).begin() >= options.ranges(i).end()) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"Indices should be non-negative and begin index should be less "
|
||||
"than the end index.");
|
||||
}
|
||||
if (options.element_only()) {
|
||||
if (options.ranges(i).end() - options.ranges(i).begin() != 1) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"Since element_only is true, all ranges should be of size 1.");
|
||||
}
|
||||
cc->Outputs().Index(i).Set<T>();
|
||||
} else {
|
||||
cc->Outputs().Index(i).Set<std::vector<T>>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
|
||||
|
||||
element_only_ = options.element_only();
|
||||
combine_outputs_ = options.combine_outputs();
|
||||
|
||||
for (const auto& range : options.ranges()) {
|
||||
ranges_.push_back({range.begin(), range.end()});
|
||||
max_range_end_ = std::max(max_range_end_, range.end());
|
||||
total_elements_ += range.end() - range.begin();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (cc->Inputs().Index(0).IsEmpty()) return ::mediapipe::OkStatus();
|
||||
|
||||
if (move_elements) {
|
||||
return ProcessMovableElements<T>(cc);
|
||||
} else {
|
||||
return ProcessCopyableElements<T>(cc);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename U, IsCopyable<U> = true>
|
||||
::mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
|
||||
// static_assert(std::is_copy_constructible<U>::value,
|
||||
// "Cannot copy non-copyable elements");
|
||||
const auto& input = cc->Inputs().Index(0).Get<std::vector<U>>();
|
||||
RET_CHECK_GE(input.size(), max_range_end_);
|
||||
if (combine_outputs_) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
output->reserve(total_elements_);
|
||||
for (int i = 0; i < ranges_.size(); ++i) {
|
||||
auto elements = absl::make_unique<std::vector<U>>(
|
||||
input.begin() + ranges_[i].first,
|
||||
input.begin() + ranges_[i].second);
|
||||
output->insert(output->end(), elements->begin(), elements->end());
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
} else {
|
||||
if (element_only_) {
|
||||
for (int i = 0; i < ranges_.size(); ++i) {
|
||||
cc->Outputs().Index(i).AddPacket(
|
||||
MakePacket<U>(input[ranges_[i].first]).At(cc->InputTimestamp()));
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < ranges_.size(); ++i) {
|
||||
auto output = absl::make_unique<std::vector<T>>(
|
||||
input.begin() + ranges_[i].first,
|
||||
input.begin() + ranges_[i].second);
|
||||
cc->Outputs().Index(i).Add(output.release(), cc->InputTimestamp());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
template <typename U, IsNotCopyable<U> = true>
|
||||
::mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
|
||||
return ::mediapipe::InternalError("Cannot copy non-copyable elements.");
|
||||
}
|
||||
|
||||
template <typename U, IsMovable<U> = true>
|
||||
::mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
|
||||
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
|
||||
cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
|
||||
if (!input_status.ok()) return input_status.status();
|
||||
std::unique_ptr<std::vector<U>> input_vector =
|
||||
std::move(input_status).ValueOrDie();
|
||||
RET_CHECK_GE(input_vector->size(), max_range_end_);
|
||||
|
||||
if (combine_outputs_) {
|
||||
auto output = absl::make_unique<std::vector<U>>();
|
||||
output->reserve(total_elements_);
|
||||
for (int i = 0; i < ranges_.size(); ++i) {
|
||||
output->insert(
|
||||
output->end(),
|
||||
std::make_move_iterator(input_vector->begin() + ranges_[i].first),
|
||||
std::make_move_iterator(input_vector->begin() + ranges_[i].second));
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||
} else {
|
||||
if (element_only_) {
|
||||
for (int i = 0; i < ranges_.size(); ++i) {
|
||||
cc->Outputs().Index(i).AddPacket(
|
||||
MakePacket<U>(std::move(input_vector->at(ranges_[i].first)))
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < ranges_.size(); ++i) {
|
||||
auto output = absl::make_unique<std::vector<T>>();
|
||||
output->insert(
|
||||
output->end(),
|
||||
std::make_move_iterator(input_vector->begin() + ranges_[i].first),
|
||||
std::make_move_iterator(input_vector->begin() +
|
||||
ranges_[i].second));
|
||||
cc->Outputs().Index(i).Add(output.release(), cc->InputTimestamp());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
template <typename U, IsNotMovable<U> = true>
|
||||
::mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
|
||||
return ::mediapipe::InternalError("Cannot move non-movable elements.");
|
||||
}
|
||||
|
||||
private:
|
||||
static ::mediapipe::Status checkRangesDontOverlap(
|
||||
const ::mediapipe::SplitVectorCalculatorOptions& options) {
|
||||
for (int i = 0; i < options.ranges_size() - 1; ++i) {
|
||||
for (int j = i + 1; j < options.ranges_size(); ++j) {
|
||||
const auto& range_0 = options.ranges(i);
|
||||
const auto& range_1 = options.ranges(j);
|
||||
if ((range_0.begin() >= range_1.begin() &&
|
||||
range_0.begin() < range_1.end()) ||
|
||||
(range_1.begin() >= range_0.begin() &&
|
||||
range_1.begin() < range_0.end())) {
|
||||
return ::mediapipe::InvalidArgumentError(
|
||||
"Ranges must be non-overlapping when using combine_outputs "
|
||||
"option.");
|
||||
}
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
std::vector<std::pair<int32, int32>> ranges_;
|
||||
int32 max_range_end_ = -1;
|
||||
int32 total_elements_ = 0;
|
||||
bool element_only_ = false;
|
||||
bool combine_outputs_ = false;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
// A Range {begin, end} specifies beginning ane ending indices to splice a
|
||||
// vector. A vector v is spliced to have elements v[begin:(end-1)], i.e., with
|
||||
// begin index inclusive and end index exclusive.
|
||||
message Range {
|
||||
optional int32 begin = 1;
|
||||
optional int32 end = 2;
|
||||
}
|
||||
|
||||
message SplitVectorCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional SplitVectorCalculatorOptions ext = 259438222;
|
||||
}
|
||||
|
||||
repeated Range ranges = 1;
|
||||
|
||||
// Specify if single element ranges should be outputted as std::vector<T> or
|
||||
// just element of type T. By default, if a range specifies only one element,
|
||||
// it is outputted as an std::vector<T>.
|
||||
optional bool element_only = 2 [default = false];
|
||||
|
||||
// Combines output elements to one vector.
|
||||
optional bool combine_outputs = 3 [default = false];
|
||||
}
|
||||
@@ -0,0 +1,694 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/core/split_vector_calculator.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
#include "tensorflow/lite/error_reporter.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
#include "tensorflow/lite/kernels/register.h"
|
||||
#include "tensorflow/lite/model.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
using ::tflite::Interpreter;
|
||||
|
||||
const int width = 1;
|
||||
const int height = 1;
|
||||
const int channels = 1;
|
||||
|
||||
class SplitTfLiteTensorVectorCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void TearDown() {
|
||||
// Note: Since the pointers contained in this vector will be cleaned up by
|
||||
// the interpreter, only ensure that the vector is cleaned up for the next
|
||||
// test.
|
||||
input_buffers_.clear();
|
||||
}
|
||||
|
||||
void PrepareTfLiteTensorVector(int vector_size) {
|
||||
ASSERT_NE(interpreter_, nullptr);
|
||||
|
||||
// Prepare input tensors.
|
||||
std::vector<int> indices(vector_size);
|
||||
for (int i = 0; i < vector_size; ++i) {
|
||||
indices[i] = i;
|
||||
}
|
||||
interpreter_->AddTensors(vector_size);
|
||||
interpreter_->SetInputs(indices);
|
||||
|
||||
input_vec_ = absl::make_unique<std::vector<TfLiteTensor>>();
|
||||
for (int i = 0; i < vector_size; ++i) {
|
||||
interpreter_->SetTensorParametersReadWrite(i, kTfLiteFloat32, "", {3},
|
||||
TfLiteQuantization());
|
||||
const int tensor_index = interpreter_->inputs()[i];
|
||||
interpreter_->ResizeInputTensor(tensor_index, {width, height, channels});
|
||||
}
|
||||
|
||||
interpreter_->AllocateTensors();
|
||||
|
||||
// Save the tensor buffer pointers for comparison after the graph runs.
|
||||
input_buffers_ = std::vector<float*>(vector_size);
|
||||
for (int i = 0; i < vector_size; ++i) {
|
||||
const int tensor_index = interpreter_->inputs()[i];
|
||||
TfLiteTensor* tensor = interpreter_->tensor(tensor_index);
|
||||
float* tensor_buffer = tensor->data.f;
|
||||
ASSERT_NE(tensor_buffer, nullptr);
|
||||
for (int j = 0; j < width * height * channels; ++j) {
|
||||
tensor_buffer[j] = i;
|
||||
}
|
||||
input_vec_->push_back(*tensor);
|
||||
input_buffers_[i] = tensor_buffer;
|
||||
}
|
||||
}
|
||||
|
||||
void ValidateVectorOutput(std::vector<Packet>& output_packets,
|
||||
int expected_elements, int input_begin_index) {
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const std::vector<TfLiteTensor>& output_vec =
|
||||
output_packets[0].Get<std::vector<TfLiteTensor>>();
|
||||
ASSERT_EQ(expected_elements, output_vec.size());
|
||||
|
||||
for (int i = 0; i < expected_elements; ++i) {
|
||||
const int expected_value = input_begin_index + i;
|
||||
const TfLiteTensor* result = &output_vec[i];
|
||||
float* result_buffer = result->data.f;
|
||||
ASSERT_NE(result_buffer, nullptr);
|
||||
ASSERT_EQ(result_buffer, input_buffers_[input_begin_index + i]);
|
||||
for (int j = 0; j < width * height * channels; ++j) {
|
||||
ASSERT_EQ(expected_value, result_buffer[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ValidateCombinedVectorOutput(std::vector<Packet>& output_packets,
|
||||
int expected_elements,
|
||||
std::vector<int>& input_begin_indices,
|
||||
std::vector<int>& input_end_indices) {
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
ASSERT_EQ(input_begin_indices.size(), input_end_indices.size());
|
||||
const std::vector<TfLiteTensor>& output_vec =
|
||||
output_packets[0].Get<std::vector<TfLiteTensor>>();
|
||||
ASSERT_EQ(expected_elements, output_vec.size());
|
||||
const int num_ranges = input_begin_indices.size();
|
||||
|
||||
int element_id = 0;
|
||||
for (int range_id = 0; range_id < num_ranges; ++range_id) {
|
||||
for (int i = input_begin_indices[range_id];
|
||||
i < input_end_indices[range_id]; ++i) {
|
||||
const int expected_value = i;
|
||||
const TfLiteTensor* result = &output_vec[element_id];
|
||||
float* result_buffer = result->data.f;
|
||||
ASSERT_NE(result_buffer, nullptr);
|
||||
ASSERT_EQ(result_buffer, input_buffers_[i]);
|
||||
for (int j = 0; j < width * height * channels; ++j) {
|
||||
ASSERT_EQ(expected_value, result_buffer[j]);
|
||||
}
|
||||
element_id++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ValidateElementOutput(std::vector<Packet>& output_packets,
|
||||
int input_begin_index) {
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
|
||||
const TfLiteTensor& result = output_packets[0].Get<TfLiteTensor>();
|
||||
float* result_buffer = result.data.f;
|
||||
ASSERT_NE(result_buffer, nullptr);
|
||||
ASSERT_EQ(result_buffer, input_buffers_[input_begin_index]);
|
||||
|
||||
const int expected_value = input_begin_index;
|
||||
for (int j = 0; j < width * height * channels; ++j) {
|
||||
ASSERT_EQ(expected_value, result_buffer[j]);
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Interpreter> interpreter_ = absl::make_unique<Interpreter>();
|
||||
std::unique_ptr<std::vector<TfLiteTensor>> input_vec_ = nullptr;
|
||||
std::vector<float*> input_buffers_;
|
||||
std::unique_ptr<CalculatorRunner> runner_ = nullptr;
|
||||
};
|
||||
|
||||
TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
|
||||
ASSERT_NE(interpreter_, nullptr);
|
||||
|
||||
PrepareTfLiteTensorVector(/*vector_size=*/5);
|
||||
ASSERT_NE(input_vec_, nullptr);
|
||||
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "tensor_in"
|
||||
output_stream: "range_0"
|
||||
output_stream: "range_1"
|
||||
output_stream: "range_2"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 1 end: 4 }
|
||||
ranges: { begin: 4 end: 5 }
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
|
||||
std::vector<Packet> range_2_packets;
|
||||
tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tensor_in", Adopt(input_vec_.release()).At(Timestamp(0))));
|
||||
// Wait until the calculator finishes processing.
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
ValidateVectorOutput(range_0_packets, /*expected_elements=*/1,
|
||||
/*input_begin_index=*/0);
|
||||
ValidateVectorOutput(range_1_packets, /*expected_elements=*/3,
|
||||
/*input_begin_index=*/1);
|
||||
ValidateVectorOutput(range_2_packets, /*expected_elements=*/1,
|
||||
/*input_begin_index=*/4);
|
||||
|
||||
// Fully close the graph at the end.
|
||||
MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
|
||||
ASSERT_NE(interpreter_, nullptr);
|
||||
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "tensor_in"
|
||||
output_stream: "range_0"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
// The graph should fail running because of an invalid range (begin == end).
|
||||
ASSERT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
|
||||
ASSERT_NE(interpreter_, nullptr);
|
||||
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "tensor_in"
|
||||
output_stream: "range_0"
|
||||
output_stream: "range_1"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
// The graph should fail running because the number of output streams does not
|
||||
// match the number of range elements in the options.
|
||||
ASSERT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
InvalidCombineOutputsMultipleOutputsTest) {
|
||||
ASSERT_NE(interpreter_, nullptr);
|
||||
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "tensor_in"
|
||||
output_stream: "range_0"
|
||||
output_stream: "range_1"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 2 end: 3 }
|
||||
combine_outputs: true
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
// The graph should fail running because the number of output streams does not
|
||||
// match the number of range elements in the options.
|
||||
ASSERT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
ASSERT_NE(interpreter_, nullptr);
|
||||
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "tensor_in"
|
||||
output_stream: "range_0"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 3 }
|
||||
ranges: { begin: 1 end: 4 }
|
||||
combine_outputs: true
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
// The graph should fail running because there are overlapping ranges.
|
||||
ASSERT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
|
||||
ASSERT_NE(interpreter_, nullptr);
|
||||
|
||||
PrepareTfLiteTensorVector(/*vector_size=*/5);
|
||||
ASSERT_NE(input_vec_, nullptr);
|
||||
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "tensor_in"
|
||||
output_stream: "range_0"
|
||||
output_stream: "range_1"
|
||||
output_stream: "range_2"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 2 end: 3 }
|
||||
ranges: { begin: 4 end: 5 }
|
||||
element_only: true
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
|
||||
std::vector<Packet> range_2_packets;
|
||||
tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tensor_in", Adopt(input_vec_.release()).At(Timestamp(0))));
|
||||
// Wait until the calculator finishes processing.
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
ValidateElementOutput(range_0_packets,
|
||||
/*input_begin_index=*/0);
|
||||
ValidateElementOutput(range_1_packets,
|
||||
/*input_begin_index=*/2);
|
||||
ValidateElementOutput(range_2_packets,
|
||||
/*input_begin_index=*/4);
|
||||
|
||||
// Fully close the graph at the end.
|
||||
MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
ASSERT_NE(interpreter_, nullptr);
|
||||
|
||||
PrepareTfLiteTensorVector(/*vector_size=*/5);
|
||||
ASSERT_NE(input_vec_, nullptr);
|
||||
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "tensor_in"
|
||||
output_stream: "range_0"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 2 end: 3 }
|
||||
ranges: { begin: 4 end: 5 }
|
||||
combine_outputs: true
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tensor_in", Adopt(input_vec_.release()).At(Timestamp(0))));
|
||||
// Wait until the calculator finishes processing.
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
std::vector<int> input_begin_indices = {0, 2, 4};
|
||||
std::vector<int> input_end_indices = {1, 3, 5};
|
||||
ValidateCombinedVectorOutput(range_0_packets, /*expected_elements=*/3,
|
||||
input_begin_indices, input_end_indices);
|
||||
|
||||
// Fully close the graph at the end.
|
||||
MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
ElementOnlyDisablesVectorOutputs) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "tensor_in"
|
||||
output_stream: "range_0"
|
||||
output_stream: "range_1"
|
||||
output_stream: "range_2"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 1 end: 4 }
|
||||
ranges: { begin: 4 end: 5 }
|
||||
element_only: true
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
ASSERT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
typedef SplitVectorCalculator<std::unique_ptr<int>, true>
|
||||
MovableSplitUniqueIntPtrCalculator;
|
||||
REGISTER_CALCULATOR(MovableSplitUniqueIntPtrCalculator);
|
||||
|
||||
class MovableSplitUniqueIntPtrCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void ValidateVectorOutput(std::vector<Packet>& output_packets,
|
||||
int expected_elements, int input_begin_index) {
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const std::vector<std::unique_ptr<int>>& output_vec =
|
||||
output_packets[0].Get<std::vector<std::unique_ptr<int>>>();
|
||||
ASSERT_EQ(expected_elements, output_vec.size());
|
||||
|
||||
for (int i = 0; i < expected_elements; ++i) {
|
||||
const int expected_value = input_begin_index + i;
|
||||
const std::unique_ptr<int>& result = output_vec[i];
|
||||
ASSERT_NE(result, nullptr);
|
||||
ASSERT_EQ(expected_value, *result);
|
||||
}
|
||||
}
|
||||
|
||||
void ValidateElementOutput(std::vector<Packet>& output_packets,
|
||||
int expected_value) {
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
const std::unique_ptr<int>& result =
|
||||
output_packets[0].Get<std::unique_ptr<int>>();
|
||||
ASSERT_NE(result, nullptr);
|
||||
ASSERT_EQ(expected_value, *result);
|
||||
}
|
||||
|
||||
void ValidateCombinedVectorOutput(std::vector<Packet>& output_packets,
|
||||
int expected_elements,
|
||||
std::vector<int>& input_begin_indices,
|
||||
std::vector<int>& input_end_indices) {
|
||||
ASSERT_EQ(1, output_packets.size());
|
||||
ASSERT_EQ(input_begin_indices.size(), input_end_indices.size());
|
||||
const std::vector<std::unique_ptr<int>>& output_vector =
|
||||
output_packets[0].Get<std::vector<std::unique_ptr<int>>>();
|
||||
ASSERT_EQ(expected_elements, output_vector.size());
|
||||
const int num_ranges = input_begin_indices.size();
|
||||
|
||||
int element_id = 0;
|
||||
for (int range_id = 0; range_id < num_ranges; ++range_id) {
|
||||
for (int i = input_begin_indices[range_id];
|
||||
i < input_end_indices[range_id]; ++i) {
|
||||
const int expected_value = i;
|
||||
const std::unique_ptr<int>& result = output_vector[element_id];
|
||||
ASSERT_NE(result, nullptr);
|
||||
ASSERT_EQ(expected_value, *result);
|
||||
++element_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "range_0"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 3 }
|
||||
ranges: { begin: 1 end: 4 }
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
// The graph should fail running because there are overlapping ranges.
|
||||
ASSERT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "range_0"
|
||||
output_stream: "range_1"
|
||||
output_stream: "range_2"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 1 end: 4 }
|
||||
ranges: { begin: 4 end: 5 }
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
|
||||
std::vector<Packet> range_2_packets;
|
||||
tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
// input_vector : {0, 1, 2, 3, 4, 5}
|
||||
std::unique_ptr<std::vector<std::unique_ptr<int>>> input_vector =
|
||||
absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
|
||||
for (int i = 0; i < 6; ++i) {
|
||||
input_vector->at(i) = absl::make_unique<int>(i);
|
||||
}
|
||||
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
|
||||
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
|
||||
ValidateVectorOutput(range_0_packets, /*expected_elements=*/1,
|
||||
/*input_begin_index=*/0);
|
||||
ValidateVectorOutput(range_1_packets, /*expected_elements=*/3,
|
||||
/*input_begin_index=*/1);
|
||||
ValidateVectorOutput(range_2_packets, /*expected_elements=*/1,
|
||||
/*input_begin_index=*/4);
|
||||
}
|
||||
|
||||
TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "range_0"
|
||||
output_stream: "range_1"
|
||||
output_stream: "range_2"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 2 end: 3 }
|
||||
ranges: { begin: 4 end: 5 }
|
||||
element_only: true
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
|
||||
std::vector<Packet> range_2_packets;
|
||||
tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
// input_vector : {0, 1, 2, 3, 4, 5}
|
||||
std::unique_ptr<std::vector<std::unique_ptr<int>>> input_vector =
|
||||
absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
|
||||
for (int i = 0; i < 6; ++i) {
|
||||
input_vector->at(i) = absl::make_unique<int>(i);
|
||||
}
|
||||
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
|
||||
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
|
||||
ValidateElementOutput(range_0_packets, /*expected_value=*/0);
|
||||
ValidateElementOutput(range_1_packets, /*expected_value=*/2);
|
||||
ValidateElementOutput(range_2_packets, /*expected_value=*/4);
|
||||
}
|
||||
|
||||
TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "range_0"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 2 end: 3 }
|
||||
ranges: { begin: 4 end: 5 }
|
||||
combine_outputs: true
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
// input_vector : {0, 1, 2, 3, 4, 5}
|
||||
std::unique_ptr<std::vector<std::unique_ptr<int>>> input_vector =
|
||||
absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
|
||||
for (int i = 0; i < 6; ++i) {
|
||||
input_vector->at(i) = absl::make_unique<int>(i);
|
||||
}
|
||||
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
|
||||
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
|
||||
std::vector<int> input_begin_indices = {0, 2, 4};
|
||||
std::vector<int> input_end_indices = {1, 3, 5};
|
||||
ValidateCombinedVectorOutput(range_0_packets, /*expected_elements=*/3,
|
||||
input_begin_indices, input_end_indices);
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// A calculator that takes a packet of an input stream and converts it to an
|
||||
// output side packet. This calculator only works under the assumption that the
|
||||
// input stream only has a single packet passing through.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "StreamToSidePacketCalculator"
|
||||
// input_stream: "stream"
|
||||
// output_side_packet: "side_packet"
|
||||
// }
|
||||
class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
|
||||
public:
|
||||
static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
cc->OutputSidePackets().Index(0).SetAny();
|
||||
return mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::Status Process(mediapipe::CalculatorContext* cc) override {
|
||||
mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
|
||||
cc->OutputSidePackets().Index(0).Set(
|
||||
packet.At(mediapipe::Timestamp::Unset()));
|
||||
return mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(StreamToSidePacketCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user