Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 MediaPipe Teamandchuoling
|
017c1dc7ea | ||
|
MediaPipe Teamandchuoling
|
a9b643e0f5 | ||
|
MediaPipe Teamandchuoling
|
ecb5b5f44a | ||
|
MediaPipe Teamandchuoling
|
7c331ad58b | ||
|
MediaPipe Teamandchuoling
|
a92cff7a60 |
+3
-1
@@ -23,6 +23,7 @@ ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
gcc-8 g++-8 \
|
||||
ca-certificates \
|
||||
curl \
|
||||
ffmpeg \
|
||||
@@ -44,6 +45,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 100 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||
RUN pip3 install --upgrade setuptools
|
||||
RUN pip3 install wheel
|
||||
RUN pip3 install future
|
||||
@@ -54,7 +56,7 @@ RUN pip3 install tf_slim
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# Install bazel
|
||||
ARG BAZEL_VERSION=3.4.1
|
||||
ARG BAZEL_VERSION=3.7.2
|
||||
RUN mkdir /bazel && \
|
||||
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
|
||||
azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||
|
||||
@@ -10,3 +10,10 @@ include requirements.txt
|
||||
recursive-include mediapipe/modules *.tflite *.txt *.binarypb
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_sneakers.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_chair.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_camera.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_3d_cup.tflite
|
||||
exclude mediapipe/modules/objectron/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite
|
||||
exclude mediapipe/modules/pose_landmark/pose_landmark_lite.tflite
|
||||
exclude mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite
|
||||
|
||||
@@ -44,7 +44,7 @@ Hair Segmentation
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
|
||||
@@ -2,16 +2,19 @@ workspace(name = "mediapipe")
|
||||
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
skylib_version = "0.9.0"
|
||||
http_archive(
|
||||
name = "bazel_skylib",
|
||||
type = "tar.gz",
|
||||
url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
|
||||
sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
|
||||
urls = [
|
||||
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
|
||||
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
|
||||
],
|
||||
sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
|
||||
)
|
||||
load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
|
||||
bazel_skylib_workspace()
|
||||
load("@bazel_skylib//lib:versions.bzl", "versions")
|
||||
versions.check(minimum_bazel_version = "3.4.0")
|
||||
|
||||
versions.check(minimum_bazel_version = "3.7.2")
|
||||
|
||||
# ABSL cpp library lts_2020_09_23
|
||||
http_archive(
|
||||
@@ -38,8 +41,8 @@ http_archive(
|
||||
|
||||
http_archive(
|
||||
name = "rules_foreign_cc",
|
||||
strip_prefix = "rules_foreign_cc-main",
|
||||
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/main.zip",
|
||||
strip_prefix = "rules_foreign_cc-0.1.0",
|
||||
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
|
||||
)
|
||||
|
||||
load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
|
||||
@@ -117,7 +120,8 @@ http_archive(
|
||||
# libyuv
|
||||
http_archive(
|
||||
name = "libyuv",
|
||||
urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"],
|
||||
# Error: operand type mismatch for `vbroadcastss' caused by commit 8a13626e42f7fdcf3a6acbb0316760ee54cda7d8.
|
||||
urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/2525698acba9bf9b701ba6b4d9584291a1f62257.tar.gz"],
|
||||
build_file = "@//third_party:libyuv.BUILD",
|
||||
)
|
||||
|
||||
@@ -304,8 +308,8 @@ http_archive(
|
||||
|
||||
# Maven dependencies.
|
||||
|
||||
RULES_JVM_EXTERNAL_TAG = "3.2"
|
||||
RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
|
||||
RULES_JVM_EXTERNAL_TAG = "4.0"
|
||||
RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169"
|
||||
|
||||
http_archive(
|
||||
name = "rules_jvm_external",
|
||||
@@ -318,7 +322,6 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
|
||||
|
||||
# Important: there can only be one maven_install rule. Add new maven deps here.
|
||||
maven_install(
|
||||
name = "maven",
|
||||
artifacts = [
|
||||
"androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
"androidx.lifecycle:lifecycle-common:2.2.0",
|
||||
@@ -334,6 +337,8 @@ maven_install(
|
||||
"androidx.test.espresso:espresso-core:3.1.1",
|
||||
"com.github.bumptech.glide:glide:4.11.0",
|
||||
"com.google.android.material:material:aar:1.0.0-rc01",
|
||||
"com.google.auto.value:auto-value:1.6.4",
|
||||
"com.google.auto.value:auto-value-annotations:1.6.4",
|
||||
"com.google.code.findbugs:jsr305:3.0.2",
|
||||
"com.google.flogger:flogger-system-backend:0.3.1",
|
||||
"com.google.flogger:flogger:0.3.1",
|
||||
@@ -343,10 +348,10 @@ maven_install(
|
||||
"org.hamcrest:hamcrest-library:1.3",
|
||||
],
|
||||
repositories = [
|
||||
"https://jcenter.bintray.com",
|
||||
"https://maven.google.com",
|
||||
"https://dl.google.com/dl/android/maven2",
|
||||
"https://repo1.maven.org/maven2",
|
||||
"https://jcenter.bintray.com",
|
||||
],
|
||||
fetch_sources = True,
|
||||
version_conflict_policy = "pinned",
|
||||
@@ -363,10 +368,10 @@ http_archive(
|
||||
],
|
||||
)
|
||||
|
||||
#Tensorflow repo should always go after the other external dependencies.
|
||||
# 2020-12-09
|
||||
_TENSORFLOW_GIT_COMMIT = "0eadbb13cef1226b1bae17c941f7870734d97f8a"
|
||||
_TENSORFLOW_SHA256= "4ae06daa5b09c62f31b7bc1f781fd59053f286dd64355830d8c2ac601b795ef0"
|
||||
# Tensorflow repo should always go after the other external dependencies.
|
||||
# 2021-04-30
|
||||
_TENSORFLOW_GIT_COMMIT = "5bd3c57ef184543d22e34e36cff9d9bea608e06d"
|
||||
_TENSORFLOW_SHA256= "9a45862834221aafacf6fb275f92b3876bc89443cbecc51be93f13839a6609f0"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
urls = [
|
||||
@@ -383,5 +388,7 @@ http_archive(
|
||||
sha256 = _TENSORFLOW_SHA256,
|
||||
)
|
||||
|
||||
load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
|
||||
tf_workspace(tf_repo_name = "org_tensorflow")
|
||||
load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
|
||||
tf_workspace3()
|
||||
load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
|
||||
tf_workspace2()
|
||||
|
||||
@@ -17,15 +17,15 @@
|
||||
# Script to build/run all MediaPipe desktop example apps (with webcam input).
|
||||
#
|
||||
# To build and run all apps and store them in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir
|
||||
# $ ./build_desktop_examples.sh -d out_dir
|
||||
# Omitting -d and the associated directory saves all generated apps in the
|
||||
# current directory.
|
||||
# To build all apps and store them in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir -b
|
||||
# $ ./build_desktop_examples.sh -d out_dir -b
|
||||
# Omitting -d and the associated directory saves all generated apps in the
|
||||
# current directory.
|
||||
# To run all apps already stored in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir -r
|
||||
# $ ./build_desktop_examples.sh -d out_dir -r
|
||||
# Omitting -d and the associated directory assumes all apps are in the current
|
||||
# directory.
|
||||
|
||||
|
||||
@@ -187,7 +187,7 @@ node {
|
||||
```
|
||||
|
||||
In the calculator implementation, inputs and outputs are also identified by tag
|
||||
name and index number. In the function below input are output are identified:
|
||||
name and index number. In the function below input and output are identified:
|
||||
|
||||
* By index number: The combined input stream is identified simply by index
|
||||
`0`.
|
||||
@@ -355,7 +355,6 @@ class PacketClonerCalculator : public CalculatorBase {
|
||||
current_[i].At(cc->InputTimestamp()));
|
||||
// Add a packet to output stream of index i a packet from inputstream i
|
||||
// with timestamp common to all present inputs
|
||||
//
|
||||
} else {
|
||||
cc->Outputs().Index(i).SetNextTimestampBound(
|
||||
cc->InputTimestamp().NextAllowedInStream());
|
||||
@@ -382,7 +381,7 @@ defined your calculator class, register it with a macro invocation
|
||||
REGISTER_CALCULATOR(calculator_class_name).
|
||||
|
||||
Below is a trivial MediaPipe graph that has 3 input streams, 1 node
|
||||
(PacketClonerCalculator) and 3 output streams.
|
||||
(PacketClonerCalculator) and 2 output streams.
|
||||
|
||||
```proto
|
||||
input_stream: "room_mic_signal"
|
||||
|
||||
@@ -110,3 +110,12 @@ Other policies are also available, implemented using a separate kind of
|
||||
component known as an InputStreamHandler.
|
||||
|
||||
See [Synchronization](synchronization.md) for more details.
|
||||
|
||||
### Realtime data streams
|
||||
|
||||
MediaPipe calculator graphs are often used to process streams of video or audio
|
||||
frames for interactive applications. Normally, each Calculator runs as soon as
|
||||
all of its input packets for a given timestamp become available. Calculators
|
||||
used in realtime graphs need to define output timestamp bounds based on input
|
||||
timestamp bounds in order to allow downstream calculators to be scheduled
|
||||
promptly. See [Realtime data streams](realtime.md) for details.
|
||||
|
||||
@@ -83,12 +83,12 @@ Below is an example of how to create a subgraph named `TwoPassThroughSubgraph`.
|
||||
output_stream: "out3"
|
||||
|
||||
node {
|
||||
calculator: "PassThroughculator"
|
||||
calculator: "PassThroughCalculator"
|
||||
input_stream: "out1"
|
||||
output_stream: "out2"
|
||||
}
|
||||
node {
|
||||
calculator: "PassThroughculator"
|
||||
calculator: "PassThroughCalculator"
|
||||
input_stream: "out2"
|
||||
output_stream: "out3"
|
||||
}
|
||||
|
||||
@@ -12,19 +12,30 @@ nav_order: 3
|
||||
{:toc}
|
||||
---
|
||||
|
||||
Each calculator is a node of of a graph. We describe how to create a new calculator, how to initialize a calculator, how to perform its calculations, input and output streams, timestamps, and options
|
||||
Calculators communicate by sending and receiving packets. Typically a single
|
||||
packet is sent along each input stream at each input timestamp. A packet can
|
||||
contain any kind of data, such as a single frame of video or a single integer
|
||||
detection count.
|
||||
|
||||
## Creating a packet
|
||||
|
||||
Packets are generally created with `MediaPipe::Adopt()` (from packet.h).
|
||||
Packets are generally created with `mediapipe::MakePacket<T>()` or
|
||||
`mediapipe::Adopt()` (from packet.h).
|
||||
|
||||
```c++
|
||||
// Create some data.
|
||||
auto data = absl::make_unique<MyDataClass>("constructor_argument");
|
||||
// Create a packet to own the data.
|
||||
Packet p = Adopt(data.release());
|
||||
// Create a packet containing some new data.
|
||||
Packet p = MakePacket<MyDataClass>("constructor_argument");
|
||||
// Make a new packet with the same data and a different timestamp.
|
||||
Packet p2 = p.At(Timestamp::PostStream());
|
||||
```
|
||||
|
||||
or:
|
||||
|
||||
```c++
|
||||
// Create some new data.
|
||||
auto data = absl::make_unique<MyDataClass>("constructor_argument");
|
||||
// Create a packet to own the data.
|
||||
Packet p = Adopt(data.release()).At(Timestamp::PostStream());
|
||||
```
|
||||
|
||||
Data within a packet is accessed with `Packet::Get<T>()`
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
---
|
||||
layout: default
|
||||
title: Processing real-time data streams
|
||||
nav_order: 6
|
||||
has_children: true
|
||||
has_toc: false
|
||||
---
|
||||
|
||||
# Processing real-time data streams
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Realtime timestamps
|
||||
|
||||
MediaPipe calculator graphs are often used to process streams of video or audio
|
||||
frames for interactive applications. The MediaPipe framework requires only that
|
||||
successive packets be assigned monotonically increasing timestamps. By
|
||||
convention, realtime calculators and graphs use the recording time or the
|
||||
presentation time of each frame as its timestamp, with each timestamp indicating
|
||||
the microseconds since `Jan/1/1970:00:00:00`. This allows packets from various
|
||||
sources to be processed in a globally consistent sequence.
|
||||
|
||||
## Realtime scheduling
|
||||
|
||||
Normally, each Calculator runs as soon as all of its input packets for a given
|
||||
timestamp become available. Normally, this happens when the calculator has
|
||||
finished processing the previous frame, and each of the calculators producing
|
||||
its inputs have finished processing the current frame. The MediaPipe scheduler
|
||||
invokes each calculator as soon as these conditions are met. See
|
||||
[Synchronization](synchronization.md) for more details.
|
||||
|
||||
## Timestamp bounds
|
||||
|
||||
When a calculator does not produce any output packets for a given timestamp, it
|
||||
can instead output a "timestamp bound" indicating that no packet will be
|
||||
produced for that timestamp. This indication is necessary to allow downstream
|
||||
calculators to run at that timestamp, even though no packet has arrived for
|
||||
certain streams for that timestamp. This is especially important for realtime
|
||||
graphs in interactive applications, where it is crucial that each calculator
|
||||
begin processing as soon as possible.
|
||||
|
||||
Consider a graph like the following:
|
||||
|
||||
```
|
||||
node {
|
||||
calculator: "A"
|
||||
input_stream: "alpha_in"
|
||||
output_stream: "alpha"
|
||||
}
|
||||
node {
|
||||
calculator: "B"
|
||||
input_stream: "alpha"
|
||||
input_stream: "foo"
|
||||
output_stream: "beta"
|
||||
}
|
||||
```
|
||||
|
||||
Suppose: at timestamp `T`, node `A` doesn't send a packet in its output stream
|
||||
`alpha`. Node `B` gets a packet in `foo` at timestamp `T` and is waiting for a
|
||||
packet in `alpha` at timestamp `T`. If `A` doesn't send `B` a timestamp bound
|
||||
update for `alpha`, `B` will keep waiting for a packet to arrive in `alpha`.
|
||||
Meanwhile, the packet queue of `foo` will accumulate packets at `T`, `T+1` and
|
||||
so on.
|
||||
|
||||
To output a packet on a stream, a calculator uses the API functions
|
||||
`CalculatorContext::Outputs` and `OutputStream::Add`. To instead output a
|
||||
timestamp bound on a stream, a calculator can use the API functions
|
||||
`CalculatorContext::Outputs` and `CalculatorContext::SetNextTimestampBound`. The
|
||||
specified bound is the lowest allowable timestamp for the next packet on the
|
||||
specified output stream. When no packet is output, a calculator will typically
|
||||
do something like:
|
||||
|
||||
```
|
||||
cc->Outputs().Tag("output_frame").SetNextTimestampBound(
|
||||
cc->InputTimestamp().NextAllowedInStream());
|
||||
```
|
||||
|
||||
The function `Timestamp::NextAllowedInStream` returns the successive timestamp.
|
||||
For example, `Timestamp(1).NextAllowedInStream() == Timestamp(2)`.
|
||||
|
||||
## Propagating timestamp bounds
|
||||
|
||||
Calculators that will be used in realtime graphs need to define output timestamp
|
||||
bounds based on input timestamp bounds in order to allow downstream calculators
|
||||
to be scheduled promptly. A common pattern is for calculators to output packets
|
||||
with the same timestamps as their input packets. In this case, simply outputting
|
||||
a packet on every call to `Calculator::Process` is sufficient to define output
|
||||
timestamp bounds.
|
||||
|
||||
However, calculators are not required to follow this common pattern for output
|
||||
timestamps, they are only required to choose monotonically increasing output
|
||||
timestamps. As a result, certain calculators must calculate timestamp bounds
|
||||
explicitly. MediaPipe provides several tools for computing appropriate timestamp
|
||||
bound for each calculator.
|
||||
|
||||
1\. **SetNextTimestampBound()** can be used to specify the timestamp bound, `t +
|
||||
1`, for an output stream.
|
||||
|
||||
```
|
||||
cc->Outputs.Tag("OUT").SetNextTimestampBound(t.NextAllowedInStream());
|
||||
```
|
||||
|
||||
Alternatively, an empty packet with timestamp `t` can be produced to specify the
|
||||
timestamp bound `t + 1`.
|
||||
|
||||
```
|
||||
cc->Outputs.Tag("OUT").Add(Packet(), t);
|
||||
```
|
||||
|
||||
The timestamp bound of an input stream is indicated by the packet or the empty
|
||||
packet on the input stream.
|
||||
|
||||
```
|
||||
Timestamp bound = cc->Inputs().Tag("IN").Value().Timestamp();
|
||||
```
|
||||
|
||||
2\. **TimestampOffset()** can be specified in order to automatically copy the
|
||||
timestamp bound from input streams to output streams.
|
||||
|
||||
```
|
||||
cc->SetTimestampOffset(0);
|
||||
```
|
||||
|
||||
This setting has the advantage of propagating timestamp bounds automatically,
|
||||
even when only timestamp bounds arrive and Calculator::Process is not invoked.
|
||||
|
||||
3\. **ProcessTimestampBounds()** can be specified in order to invoke
|
||||
`Calculator::Process` for each new "settled timestamp", where the "settled
|
||||
timestamp" is the new highest timestamp below the current timestamp bounds.
|
||||
Without `ProcessTimestampBounds()`, `Calculator::Process` is invoked only with
|
||||
one or more arriving packets.
|
||||
|
||||
```
|
||||
cc->SetProcessTimestampBounds(true);
|
||||
```
|
||||
|
||||
This setting allows a calculator to perform its own timestamp bounds calculation
|
||||
and propagation, even when only input timestamps are updated. It can be used to
|
||||
replicate the effect of `TimestampOffset()`, but it can also be used to
|
||||
calculate a timestamp bound that takes into account additional factors.
|
||||
|
||||
For example, in order to replicate `SetTimestampOffset(0)`, a calculator could
|
||||
do the following:
|
||||
|
||||
```
|
||||
absl::Status Open(CalculatorContext* cc) {
|
||||
cc->SetProcessTimestampBounds(true);
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) {
|
||||
cc->Outputs.Tag("OUT").SetNextTimestampBound(
|
||||
cc->InputTimestamp().NextAllowedInStream());
|
||||
}
|
||||
```
|
||||
|
||||
## Scheduling of Calculator::Open and Calculator::Close
|
||||
|
||||
`Calculator::Open` is invoked when all required input side-packets have been
|
||||
produced. Input side-packets can be provided by the enclosing application or by
|
||||
"side-packet calculators" inside the graph. Side-packets can be specified from
|
||||
outside the graph using the API's `CalculatorGraph::Initialize` and
|
||||
`CalculatorGraph::StartRun`. Side packets can be specified by calculators within
|
||||
the graph using `CalculatorGraphConfig::OutputSidePackets` and
|
||||
`OutputSidePacket::Set`.
|
||||
|
||||
Calculator::Close is invoked when all of the input streams have become `Done` by
|
||||
being closed or reaching timestamp bound `Timestamp::Done`.
|
||||
|
||||
**Note:** If the graph finishes all pending calculator execution and becomes
|
||||
`Done`, before some streams become `Done`, then MediaPipe will invoke the
|
||||
remaining calls to `Calculator::Close`, so that every calculator can produce its
|
||||
final outputs.
|
||||
|
||||
The use of `TimestampOffset` has some implications for `Calculator::Close`. A
|
||||
calculator specifying `SetTimestampOffset(0)` will by design signal that all of
|
||||
its output streams have reached `Timestamp::Done` when all of its input streams
|
||||
have reached `Timestamp::Done`, and therefore no further outputs are possible.
|
||||
This prevents such a calculator from emitting any packets during
|
||||
`Calculator::Close`. If a calculator needs to produce a summary packet during
|
||||
`Calculator::Close`, `Calculator::Process` must specify timestamp bounds such
|
||||
that at least one timestamp (such as `Timestamp::Max`) remains available during
|
||||
`Calculator::Close`. This means that such a calculator normally cannot rely upon
|
||||
`SetTimestampOffset(0)` and must instead specify timestamp bounds explicitly
|
||||
using `SetNextTimestampBounds()`.
|
||||
@@ -28,7 +28,7 @@ Gradle.
|
||||
* Install MediaPipe following these [instructions](./install.md).
|
||||
* Setup Java Runtime.
|
||||
* Setup Android SDK release 28.0.3 and above.
|
||||
* Setup Android NDK r18b and above.
|
||||
* Setup Android NDK version between 18 and 21.
|
||||
|
||||
MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
|
||||
below for Android Studio setup). However, if you prefer using MediaPipe without
|
||||
@@ -57,7 +57,7 @@ Please verify all the necessary packages are installed.
|
||||
* Android SDK Build-Tools 28 or 29
|
||||
* Android SDK Platform-Tools 28 or 29
|
||||
* Android SDK Tools 26.1.1
|
||||
* Android NDK 17c or above
|
||||
* Android NDK 19c or above
|
||||
|
||||
### Option 1: Build with Bazel in Command Line
|
||||
|
||||
@@ -111,7 +111,7 @@ app:
|
||||
* Verify that Android SDK Build-Tools 28 or 29 is installed.
|
||||
* Verify that Android SDK Platform-Tools 28 or 29 is installed.
|
||||
* Verify that Android SDK Tools 26.1.1 is installed.
|
||||
* Verify that Android NDK 17c or above is installed.
|
||||
* Verify that Android NDK 19c or above is installed.
|
||||
* Take note of the Android NDK Location, e.g.,
|
||||
`/usr/local/home/Android/Sdk/ndk-bundle` or
|
||||
`/usr/local/home/Android/Sdk/ndk/20.0.5594570`.
|
||||
|
||||
@@ -37,7 +37,7 @@ each project.
|
||||
load("//mediapipe/java/com/google/mediapipe:mediapipe_aar.bzl", "mediapipe_aar")
|
||||
|
||||
mediapipe_aar(
|
||||
name = "mp_face_detection_aar",
|
||||
name = "mediapipe_face_detection",
|
||||
calculators = ["//mediapipe/graphs/face_detection:mobile_calculators"],
|
||||
)
|
||||
```
|
||||
@@ -45,26 +45,29 @@ each project.
|
||||
2. Run the Bazel build command to generate the AAR.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
|
||||
--fat_apk_cpu=arm64-v8a,armeabi-v7a --strip=ALWAYS \
|
||||
//path/to/the/aar/build/file:aar_name
|
||||
bazel build -c opt --strip=ALWAYS \
|
||||
--host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
|
||||
--fat_apk_cpu=arm64-v8a,armeabi-v7a \
|
||||
//path/to/the/aar/build/file:aar_name.aar
|
||||
```
|
||||
|
||||
For the face detection AAR target we made in the step 1, run:
|
||||
For the face detection AAR target we made in step 1, run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
|
||||
//mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar
|
||||
bazel build -c opt --strip=ALWAYS \
|
||||
--host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
|
||||
--fat_apk_cpu=arm64-v8a,armeabi-v7a \
|
||||
//mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mediapipe_face_detection.aar
|
||||
|
||||
# It should print:
|
||||
# Target //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar up-to-date:
|
||||
# bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
|
||||
# Target //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mediapipe_face_detection.aar up-to-date:
|
||||
# bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mediapipe_face_detection.aar
|
||||
```
|
||||
|
||||
3. (Optional) Save the AAR to your preferred location.
|
||||
|
||||
```bash
|
||||
cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
|
||||
cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mediapipe_face_detection.aar
|
||||
/absolute/path/to/your/preferred/location
|
||||
```
|
||||
|
||||
@@ -75,7 +78,7 @@ each project.
|
||||
2. Copy the AAR into app/libs.
|
||||
|
||||
```bash
|
||||
cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
|
||||
cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mediapipe_face_detection.aar
|
||||
/path/to/your/app/libs/
|
||||
```
|
||||
|
||||
@@ -92,29 +95,14 @@ each project.
|
||||
[the face detection tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite).
|
||||
|
||||
```bash
|
||||
bazel build -c opt mediapipe/mediapipe/graphs/face_detection:mobile_gpu_binary_graph
|
||||
cp bazel-bin/mediapipe/graphs/face_detection/mobile_gpu.binarypb /path/to/your/app/src/main/assets/
|
||||
bazel build -c opt mediapipe/graphs/face_detection:face_detection_mobile_gpu_binary_graph
|
||||
cp bazel-bin/mediapipe/graphs/face_detection/face_detection_mobile_gpu.binarypb /path/to/your/app/src/main/assets/
|
||||
cp mediapipe/modules/face_detection/face_detection_front.tflite /path/to/your/app/src/main/assets/
|
||||
```
|
||||
|
||||
|
||||
|
||||
4. Make app/src/main/jniLibs and copy OpenCV JNI libraries into
|
||||
app/src/main/jniLibs.
|
||||
|
||||
MediaPipe depends on OpenCV, you will need to copy the precompiled OpenCV so
|
||||
files into app/src/main/jniLibs. You can download the official OpenCV
|
||||
Android SDK from
|
||||
[here](https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip)
|
||||
and run:
|
||||
|
||||
```bash
|
||||
cp -R ~/Downloads/OpenCV-android-sdk/sdk/native/libs/arm* /path/to/your/app/src/main/jniLibs/
|
||||
```
|
||||
|
||||
|
||||
|
||||
5. Modify app/build.gradle to add MediaPipe dependencies and MediaPipe AAR.
|
||||
4. Modify app/build.gradle to add MediaPipe dependencies and MediaPipe AAR.
|
||||
|
||||
```
|
||||
dependencies {
|
||||
@@ -136,10 +124,14 @@ each project.
|
||||
implementation "androidx.camera:camera-core:$camerax_version"
|
||||
implementation "androidx.camera:camera-camera2:$camerax_version"
|
||||
implementation "androidx.camera:camera-lifecycle:$camerax_version"
|
||||
// AutoValue
|
||||
def auto_value_version = "1.6.4"
|
||||
implementation "com.google.auto.value:auto-value-annotations:$auto_value_version"
|
||||
annotationProcessor "com.google.auto.value:auto-value:$auto_value_version"
|
||||
}
|
||||
```
|
||||
|
||||
6. Follow our Android app examples to use MediaPipe in Android Studio for your
|
||||
5. Follow our Android app examples to use MediaPipe in Android Studio for your
|
||||
use case. If you are looking for an example, a face detection example can be
|
||||
found
|
||||
[here](https://github.com/jiuqiant/mediapipe_face_detection_aar_example) and
|
||||
|
||||
@@ -25,25 +25,11 @@ install --user six`.
|
||||
|
||||
## Installing on Debian and Ubuntu
|
||||
|
||||
1. Install Bazel.
|
||||
1. Install Bazelisk.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
|
||||
to install Bazel 3.4 or higher.
|
||||
|
||||
For Nvidia Jetson and Raspberry Pi devices with aarch64 Linux, Bazel needs
|
||||
to be built from source:
|
||||
|
||||
```bash
|
||||
# For Bazel 3.4.1
|
||||
mkdir $HOME/bazel-3.4.1
|
||||
cd $HOME/bazel-3.4.1
|
||||
wget https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-dist.zip
|
||||
sudo apt-get install build-essential openjdk-8-jdk python zip unzip
|
||||
unzip bazel-3.4.1-dist.zip
|
||||
env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
|
||||
sudo cp output/bazel /usr/local/bin/
|
||||
```
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
2. Checkout MediaPipe repository.
|
||||
|
||||
@@ -207,11 +193,11 @@ build issues.
|
||||
|
||||
**Disclaimer**: Running MediaPipe on CentOS is experimental.
|
||||
|
||||
1. Install Bazel.
|
||||
1. Install Bazelisk.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
|
||||
to install Bazel 3.4 or higher.
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
2. Checkout MediaPipe repository.
|
||||
|
||||
@@ -336,11 +322,11 @@ build issues.
|
||||
* Install [Xcode](https://developer.apple.com/xcode/) and its Command Line
|
||||
Tools by `xcode-select --install`.
|
||||
|
||||
2. Install Bazel.
|
||||
2. Install Bazelisk.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
|
||||
to install Bazel 3.4 or higher.
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
3. Checkout MediaPipe repository.
|
||||
|
||||
@@ -353,7 +339,7 @@ build issues.
|
||||
4. Install OpenCV and FFmpeg.
|
||||
|
||||
Option 1. Use HomeBrew package manager tool to install the pre-compiled
|
||||
OpenCV 3.4.5 libraries. FFmpeg will be installed via OpenCV.
|
||||
OpenCV 3 libraries. FFmpeg will be installed via OpenCV.
|
||||
|
||||
```bash
|
||||
$ brew install opencv@3
|
||||
@@ -484,29 +470,36 @@ next section.
|
||||
|
||||
4. Install Visual C++ Build Tools 2019 and WinSDK
|
||||
|
||||
Go to https://visualstudio.microsoft.com/visual-cpp-build-tools, download
|
||||
build tools, and install Microsoft Visual C++ 2019 Redistributable and
|
||||
Microsoft Build Tools 2019.
|
||||
Go to
|
||||
[the VisualStudio website](https://visualstudio.microsoft.com/visual-cpp-build-tools),
|
||||
download build tools, and install Microsoft Visual C++ 2019 Redistributable
|
||||
and Microsoft Build Tools 2019.
|
||||
|
||||
Download the WinSDK from
|
||||
https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/ and
|
||||
install.
|
||||
[the official MicroSoft website](https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/)
|
||||
and install.
|
||||
|
||||
5. Install Bazel and add the location of the Bazel executable to the `%PATH%`
|
||||
environment variable.
|
||||
5. Install Bazel or Bazelisk and add the location of the Bazel executable to
|
||||
the `%PATH%` environment variable.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
|
||||
to install Bazel 3.4 or higher.
|
||||
Option 1. Follow
|
||||
[the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
|
||||
to install Bazel 3.7.2 or higher.
|
||||
|
||||
6. Set Bazel variables.
|
||||
Option 2. Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
6. Set Bazel variables. Learn more details about
|
||||
["Build on Windows"](https://docs.bazel.build/versions/master/windows.html#build-c-with-msvc)
|
||||
in the Bazel official documentation.
|
||||
|
||||
```
|
||||
# Find the exact paths and version numbers from your local version.
|
||||
# Please find the exact paths and version numbers from your local version.
|
||||
C:\> set BAZEL_VS=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools
|
||||
C:\> set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC
|
||||
C:\> set BAZEL_VC_FULL_VERSION=14.25.28610
|
||||
C:\> set BAZEL_WINSDK_FULL_VERSION=10.1.18362.1
|
||||
C:\> set BAZEL_VC_FULL_VERSION=<Your local VC version>
|
||||
C:\> set BAZEL_WINSDK_FULL_VERSION=<Your local WinSDK version>
|
||||
```
|
||||
|
||||
7. Checkout MediaPipe repository.
|
||||
@@ -593,19 +586,11 @@ cameras. Alternatively, you use a video file as input.
|
||||
username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
|
||||
```
|
||||
|
||||
5. Install Bazel.
|
||||
5. Install Bazelisk.
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
|
||||
https://storage.googleapis.com/bazel/3.4.1/release/bazel-3.4.1-installer-linux-x86_64.sh && \
|
||||
sudo mkdir -p /usr/local/bazel/3.4.1 && \
|
||||
chmod 755 bazel-3.4.1-installer-linux-x86_64.sh && \
|
||||
sudo ./bazel-3.4.1-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.4.1 && \
|
||||
source /usr/local/bazel/3.4.1/lib/bazel/bin/bazel-complete.bash
|
||||
|
||||
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.4.1/lib/bazel/bin/bazel version && \
|
||||
alias bazel='/usr/local/bazel/3.4.1/lib/bazel/bin/bazel'
|
||||
```
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
|
||||
to install Bazelisk.
|
||||
|
||||
6. Checkout MediaPipe repository.
|
||||
|
||||
@@ -753,7 +738,7 @@ common build issues.
|
||||
root@bca08b91ff63:/mediapipe# bash ./setup_android_sdk_and_ndk.sh
|
||||
|
||||
# Should print:
|
||||
# Android NDK is now installed. Consider setting $ANDROID_NDK_HOME environment variable to be /root/Android/Sdk/ndk-bundle/android-ndk-r18b
|
||||
# Android NDK is now installed. Consider setting $ANDROID_NDK_HOME environment variable to be /root/Android/Sdk/ndk-bundle/android-ndk-r19c
|
||||
# Set android_ndk_repository and android_sdk_repository in WORKSPACE
|
||||
# Done
|
||||
|
||||
|
||||
@@ -72,9 +72,9 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
|
||||
[Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection
|
||||
[H-npm]: https://www.npmjs.com/package/@mediapipe/hands
|
||||
[P-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[draw-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[cam-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[draw-npm]: https://www.npmjs.com/package/@mediapipe/drawing_utils
|
||||
[cam-npm]: https://www.npmjs.com/package/@mediapipe/camera_utils
|
||||
[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/control_utils
|
||||
[Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic
|
||||
[F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh
|
||||
[Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection
|
||||
|
||||
@@ -26,7 +26,7 @@ You can, for instance, activate a Python virtual environment:
|
||||
$ python3 -m venv mp_env && source mp_env/bin/activate
|
||||
```
|
||||
|
||||
Install MediaPipe Python package and start Python intepreter:
|
||||
Install MediaPipe Python package and start Python interpreter:
|
||||
|
||||
```bash
|
||||
(mp_env)$ pip install mediapipe
|
||||
|
||||
@@ -97,6 +97,49 @@ linux_opencv/macos_opencv/windows_opencv.BUILD files for your local opencv
|
||||
libraries. [This GitHub issue](https://github.com/google/mediapipe/issues/666)
|
||||
may also help.
|
||||
|
||||
## Python pip install failure
|
||||
|
||||
The error message:
|
||||
|
||||
```
|
||||
ERROR: Could not find a version that satisfies the requirement mediapipe
|
||||
ERROR: No matching distribution found for mediapipe
|
||||
```
|
||||
|
||||
after running `pip install mediapipe` usually indicates that there is no qualified MediaPipe Python for your system.
|
||||
Please note that MediaPipe Python PyPI officially supports the **64-bit**
|
||||
version of Python 3.7 and above on the following OS:
|
||||
|
||||
- x86_64 Linux
|
||||
- x86_64 macOS 10.15+
|
||||
- amd64 Windows
|
||||
|
||||
If the OS is currently supported and you still see this error, please make sure
|
||||
that both the Python and pip binary are for Python 3.7 and above. Otherwise,
|
||||
please consider building the MediaPipe Python package locally by following the
|
||||
instructions [here](python.md#building-mediapipe-python-package).
|
||||
|
||||
## Python DLL load failure on Windows
|
||||
|
||||
The error message:
|
||||
|
||||
```
|
||||
ImportError: DLL load failed: The specified module could not be found
|
||||
```
|
||||
|
||||
usually indicates that the local Windows system is missing Visual C++
|
||||
redistributable packages and/or Visual C++ runtime DLLs. This can be solved by
|
||||
either installing the official
|
||||
[vc_redist.x64.exe](https://support.microsoft.com/en-us/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0)
|
||||
or installing the "msvc-runtime" Python package by running
|
||||
|
||||
```bash
|
||||
$ python -m pip install msvc-runtime
|
||||
```
|
||||
|
||||
Please note that the "msvc-runtime" Python package is not released or maintained
|
||||
by Microsoft.
|
||||
|
||||
## Native method not found
|
||||
|
||||
The error message:
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 35 KiB After Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 29 KiB After Width: | Height: | Size: 42 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.3 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 56 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 6.9 MiB |
+1
-1
@@ -44,7 +44,7 @@ Hair Segmentation
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
|
||||
@@ -77,7 +77,8 @@ Supported configuration options:
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_face_detction = mp.solutions.face_detection
|
||||
mp_face_detection = mp.solutions.face_detection
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
|
||||
# For static images:
|
||||
with mp_face_detection.FaceDetection(
|
||||
@@ -183,8 +184,8 @@ function onResults(results) {
|
||||
canvasCtx.restore();
|
||||
}
|
||||
|
||||
const faceDetection = new Objectron({locateFile: (file) => {
|
||||
return `https://cdn.jsdelivr.net/npm/@mediapipe/objectr[email protected]/${file}`;
|
||||
const faceDetection = new FaceDetection({locateFile: (file) => {
|
||||
return `https://cdn.jsdelivr.net/npm/@mediapipe/face_detecti[email protected]/${file}`;
|
||||
}});
|
||||
faceDetection.setOptions({
|
||||
minDetectionConfidence: 0.5
|
||||
|
||||
+10
-11
@@ -135,12 +135,11 @@ another detection until it loses track, on reducing computation and latency. If
|
||||
set to `true`, person detection runs every input image, ideal for processing a
|
||||
batch of static, possibly unrelated, images. Default to `false`.
|
||||
|
||||
#### upper_body_only
|
||||
#### model_complexity
|
||||
|
||||
If set to `true`, the solution outputs only the 25 upper-body pose landmarks
|
||||
(535 in total) instead of the full set of 33 pose landmarks (543 in total). Note
|
||||
that upper-body-only prediction may be more accurate for use cases where the
|
||||
lower-body parts are mostly out of view. Default to `false`.
|
||||
Complexity of the pose landmark model: `0`, `1` or `2`. Landmark accuracy as
|
||||
well as inference latency generally go up with the model complexity. Default to
|
||||
`1`.
|
||||
|
||||
#### smooth_landmarks
|
||||
|
||||
@@ -207,7 +206,7 @@ install MediaPipe Python package, then learn more in the companion
|
||||
Supported configuration options:
|
||||
|
||||
* [static_image_mode](#static_image_mode)
|
||||
* [upper_body_only](#upper_body_only)
|
||||
* [model_complexity](#model_complexity)
|
||||
* [smooth_landmarks](#smooth_landmarks)
|
||||
* [min_detection_confidence](#min_detection_confidence)
|
||||
* [min_tracking_confidence](#min_tracking_confidence)
|
||||
@@ -219,7 +218,9 @@ mp_drawing = mp.solutions.drawing_utils
|
||||
mp_holistic = mp.solutions.holistic
|
||||
|
||||
# For static images:
|
||||
with mp_holistic.Holistic(static_image_mode=True) as holistic:
|
||||
with mp_holistic.Holistic(
|
||||
static_image_mode=True,
|
||||
model_complexity=2) as holistic:
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
image_height, image_width, _ = image.shape
|
||||
@@ -240,8 +241,6 @@ with mp_holistic.Holistic(static_image_mode=True) as holistic:
|
||||
annotated_image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
# Use mp_holistic.UPPER_BODY_POSE_CONNECTIONS for drawing below when
|
||||
# upper_body_only is set to True.
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
@@ -291,7 +290,7 @@ and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
* [upperBodyOnly](#upper_body_only)
|
||||
* [modelComplexity](#model_complexity)
|
||||
* [smoothLandmarks](#smooth_landmarks)
|
||||
* [minDetectionConfidence](#min_detection_confidence)
|
||||
* [minTrackingConfidence](#min_tracking_confidence)
|
||||
@@ -348,7 +347,7 @@ const holistic = new Holistic({locateFile: (file) => {
|
||||
return `https://cdn.jsdelivr.net/npm/@mediapipe/holistic/${file}`;
|
||||
}});
|
||||
holistic.setOptions({
|
||||
upperBodyOnly: false,
|
||||
modelComplexity: 1,
|
||||
smoothLandmarks: true,
|
||||
minDetectionConfidence: 0.5,
|
||||
minTrackingConfidence: 0.5
|
||||
|
||||
@@ -15,10 +15,10 @@ nav_order: 30
|
||||
### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
|
||||
|
||||
* Face detection model for front-facing/selfie camera:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite),
|
||||
[TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
|
||||
* Face detection model for back-facing camera:
|
||||
[TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
|
||||
[TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_back.tflite)
|
||||
* [Model card](https://mediapipe.page.link/blazeface-mc)
|
||||
|
||||
### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)
|
||||
@@ -49,10 +49,10 @@ nav_order: 30
|
||||
|
||||
* Pose detection model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
|
||||
* Full-body pose landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full_body.tflite)
|
||||
* Upper-body pose landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
|
||||
* Pose landmark model:
|
||||
[TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_lite.tflite),
|
||||
[TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full.tflite),
|
||||
[TFLite model (heavy)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite)
|
||||
* [Model card](https://mediapipe.page.link/blazepose-mc)
|
||||
|
||||
### [Holistic](https://google.github.io/mediapipe/solutions/holistic)
|
||||
|
||||
+40
-11
@@ -358,15 +358,17 @@ cap.release()
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
[Android](../getting_started/android.md) and [iOS](../getting_started/ios.md) on
|
||||
how to build MediaPipe examples.
|
||||
[Android](../getting_started/android.md), [iOS](../getting_started/ios.md), and
|
||||
[desktop](../getting_started/cpp.md) on how to build MediaPipe examples.
|
||||
|
||||
Note: To visualize a graph, copy the graph and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
### Two-stage Objectron
|
||||
### Mobile
|
||||
|
||||
#### Two-stage Objectron
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
|
||||
@@ -404,7 +406,7 @@ to visualize its associated subgraphs, please see
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
### Single-stage Objectron
|
||||
#### Single-stage Objectron
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
|
||||
@@ -428,7 +430,7 @@ to visualize its associated subgraphs, please see
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
### Assets
|
||||
#### Assets
|
||||
|
||||
Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc) using a parsing of the sequenced .obj file
|
||||
format into a custom .uuu format. This can be done for user assets as follows:
|
||||
@@ -449,9 +451,35 @@ Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](http
|
||||
> single .uuu animation file, using the order given by sorting the filenames alphanumerically. Also the ObjParser directory inputs must be given as
|
||||
> absolute paths, not relative paths. See parser utility library at [`mediapipe/graphs/object_detection_3d/obj_parser/`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/obj_parser/) for more details.
|
||||
|
||||
### Coordinate Systems
|
||||
|
||||
#### Object Coordinate
|
||||
### Desktop
|
||||
|
||||
To build the application, run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/object_detection_3d:objectron_cpu
|
||||
```
|
||||
|
||||
To run the application, replace `<input video path>` and `<output video path>`
|
||||
in the command below with your own paths, and `<landmark model path>` and
|
||||
`<allowed labels>` with the following:
|
||||
|
||||
Category | `<landmark model path>` | `<allowed labels>`
|
||||
:------- | :-------------------------------------------------------------------------- | :-----------------
|
||||
Shoe | mediapipe/modules/objectron/object_detection_3d_sneakers.tflite | Footwear
|
||||
Chair | mediapipe/modules/objectron/object_detection_3d_chair.tflite | Chair
|
||||
Cup | mediapipe/modules/objectron/object_detection_3d_cup.tflite | Mug
|
||||
Camera | mediapipe/modules/objectron/object_detection_3d_camera.tflite | Camera
|
||||
|
||||
```
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection_3d/objectron_cpu \
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt \
|
||||
--input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>,box_landmark_model_path=<landmark model path>,allowed_labels=<allowed labels>
|
||||
```
|
||||
|
||||
## Coordinate Systems
|
||||
|
||||
### Object Coordinate
|
||||
|
||||
Each object has its object coordinate frame. We use the below object coordinate
|
||||
definition, with `+x` pointing right, `+y` pointing up and `+z` pointing front,
|
||||
@@ -459,7 +487,7 @@ origin is at the center of the 3D bounding box.
|
||||
|
||||
|
||||
|
||||
#### Camera Coordinate
|
||||
### Camera Coordinate
|
||||
|
||||
A 3D object is parameterized by its `scale` and `rotation`, `translation` with
|
||||
regard to the camera coordinate frame. In this API we use the below camera
|
||||
@@ -476,7 +504,7 @@ camera frame by applying `rotation` and `translation`:
|
||||
landmarks_3d = rotation * scale * unit_box + translation
|
||||
```
|
||||
|
||||
#### NDC Space
|
||||
### NDC Space
|
||||
|
||||
In this API we use
|
||||
[NDC(normalized device coordinates)](http://www.songho.ca/opengl/gl_projectionmatrix.html)
|
||||
@@ -495,7 +523,7 @@ y_ndc = -fy * Y / Z + py
|
||||
z_ndc = 1 / Z
|
||||
```
|
||||
|
||||
#### Pixel Space
|
||||
### Pixel Space
|
||||
|
||||
In this API we set upper-left coner of an image as the origin of pixel
|
||||
coordinate. One can convert from NDC to pixel space as follows:
|
||||
@@ -532,10 +560,11 @@ py = -py_pixel * 2.0 / image_height + 1.0
|
||||
[Announcing the Objectron Dataset](https://ai.googleblog.com/2020/11/announcing-objectron-dataset.html)
|
||||
* Google AI Blog:
|
||||
[Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
|
||||
* Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in CVPR 2021
|
||||
* Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
|
||||
Shape Supervision](https://arxiv.org/abs/2003.03522)
|
||||
* Paper:
|
||||
[Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
|
||||
([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
|
||||
([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth Workshop on Computer Vision for AR/VR, CVPR 2020
|
||||
* [Models and model cards](./models.md#objectron)
|
||||
* [Python Colab](https://mediapipe.page.link/objectron_py_colab)
|
||||
|
||||
+56
-170
@@ -2,6 +2,8 @@
|
||||
layout: default
|
||||
title: Pose
|
||||
parent: Solutions
|
||||
has_children: true
|
||||
has_toc: false
|
||||
nav_order: 5
|
||||
---
|
||||
|
||||
@@ -21,16 +23,14 @@ nav_order: 5
|
||||
## Overview
|
||||
|
||||
Human pose estimation from video plays a critical role in various applications
|
||||
such as
|
||||
[quantifying physical exercises](#pose-classification-and-repetition-counting),
|
||||
sign language recognition, and full-body gesture control. For example, it can
|
||||
form the basis for yoga, dance, and fitness applications. It can also enable the
|
||||
such as [quantifying physical exercises](./pose_classification.md), sign
|
||||
language recognition, and full-body gesture control. For example, it can form
|
||||
the basis for yoga, dance, and fitness applications. It can also enable the
|
||||
overlay of digital content and information on top of the physical world in
|
||||
augmented reality.
|
||||
|
||||
MediaPipe Pose is a ML solution for high-fidelity body pose tracking, inferring
|
||||
33 3D landmarks on the whole body (or 25 upper-body landmarks) from RGB video
|
||||
frames utilizing our
|
||||
33 3D landmarks on the whole body from RGB video frames utilizing our
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
research that also powers the
|
||||
[ML Kit Pose Detection API](https://developers.google.com/ml-kit/vision/pose-detection).
|
||||
@@ -39,9 +39,9 @@ environments for inference, whereas our method achieves real-time performance on
|
||||
most modern [mobile phones](#mobile), [desktops/laptops](#desktop), in
|
||||
[python](#python-solution-api) and even on the [web](#javascript-solution-api).
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------: |
|
||||
*Fig 1. Example of MediaPipe Pose for upper-body pose tracking.* |
|
||||
 |
|
||||
:----------------------------------------------------------------------: |
|
||||
*Fig 1. Example of MediaPipe Pose for pose tracking.* |
|
||||
|
||||
## ML Pipeline
|
||||
|
||||
@@ -76,6 +76,36 @@ Note: To visualize a graph, copy the graph and paste it into
|
||||
to visualize its associated subgraphs, please see
|
||||
[visualizer documentation](../tools/visualizer.md).
|
||||
|
||||
## Pose Estimation Quality
|
||||
|
||||
To evaluate the quality of our [models](./models.md#pose) against other
|
||||
well-performing publicly available solutions, we use three different validation
|
||||
datasets, representing different verticals: Yoga, Dance and HIIT. Each image
|
||||
contains only a single person located 2-4 meters from the camera. To be
|
||||
consistent with other solutions, we perform evaluation only for 17 keypoints
|
||||
from [COCO topology](https://cocodataset.org/#keypoints-2020).
|
||||
|
||||
Method | Yoga <br/> [`mAP`] | Yoga <br/> [`[email protected]`] | Dance <br/> [`mAP`] | Dance <br/> [`[email protected]`] | HIIT <br/> [`mAP`] | HIIT <br/> [`[email protected]`]
|
||||
----------------------------------------------------------------------------------------------------- | -----------------: | ---------------------: | ------------------: | ----------------------: | -----------------: | ---------------------:
|
||||
BlazePose.Heavy | 68.1 | **96.4** | 73.0 | **97.2** | 74.0 | **97.5**
|
||||
BlazePose.Full | 62.6 | **95.5** | 67.4 | **96.3** | 68.0 | **95.7**
|
||||
BlazePose.Lite | 45.0 | **90.2** | 53.6 | **92.5** | 53.8 | **93.5**
|
||||
[AlphaPose.ResNet50](https://github.com/MVIG-SJTU/AlphaPose) | 63.4 | **96.0** | 57.8 | **95.5** | 63.4 | **96.0**
|
||||
[Apple.Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 32.8 | **82.7** | 36.4 | **91.4** | 44.5 | **88.6**
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------: |
|
||||
*Fig 2. Quality evaluation in [`[email protected]`].* |
|
||||
|
||||
We designed our models specifically for live perception use cases, so all of
|
||||
them work in real-time on the majority of modern devices.
|
||||
|
||||
Method | Latency <br/> Pixel 3 [TFLite GPU](https://www.tensorflow.org/lite/performance/gpu_advanced) | Latency <br/> MacBook Pro (15-inch 2017)
|
||||
--------------- | -------------------------------------------------------------------------------------------: | ---------------------------------------:
|
||||
BlazePose.Heavy | 53 ms | 38 ms
|
||||
BlazePose.Full | 25 ms | 27 ms
|
||||
BlazePose.Lite | 20 ms | 25 ms
|
||||
|
||||
## Models
|
||||
|
||||
### Person/pose Detection Model (BlazePose Detector)
|
||||
@@ -92,15 +122,12 @@ hip midpoints.
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------------------------------: |
|
||||
*Fig 2. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
|
||||
*Fig 3. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
|
||||
|
||||
### Pose Landmark Model (BlazePose GHUM 3D)
|
||||
|
||||
The landmark model in MediaPipe Pose comes in two versions: a full-body model
|
||||
that predicts the location of 33 pose landmarks (see figure below), and an
|
||||
upper-body version that only predicts the first 25. The latter may be more
|
||||
accurate than the former in scenarios where the lower-body parts are mostly out
|
||||
of view.
|
||||
The landmark model in MediaPipe Pose predicts the location of 33 pose landmarks
|
||||
(see figure below).
|
||||
|
||||
Please find more detail in the
|
||||
[BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html),
|
||||
@@ -110,7 +137,7 @@ this [paper](https://arxiv.org/abs/2006.10204) and
|
||||
|
||||
 |
|
||||
:----------------------------------------------------------------------------------------------: |
|
||||
*Fig 3. 33 pose landmarks.* |
|
||||
*Fig 4. 33 pose landmarks.* |
|
||||
|
||||
## Solution APIs
|
||||
|
||||
@@ -128,12 +155,11 @@ until it loses track, on reducing computation and latency. If set to `true`,
|
||||
person detection runs every input image, ideal for processing a batch of static,
|
||||
possibly unrelated, images. Default to `false`.
|
||||
|
||||
#### upper_body_only
|
||||
#### model_complexity
|
||||
|
||||
If set to `true`, the solution outputs only the 25 upper-body pose landmarks.
|
||||
Otherwise, it outputs the full set of 33 pose landmarks. Note that
|
||||
upper-body-only prediction may be more accurate for use cases where the
|
||||
lower-body parts are mostly out of view. Default to `false`.
|
||||
Complexity of the pose landmark model: `0`, `1` or `2`. Landmark accuracy as
|
||||
well as inference latency generally go up with the model complexity. Default to
|
||||
`1`.
|
||||
|
||||
#### smooth_landmarks
|
||||
|
||||
@@ -169,9 +195,6 @@ A list of pose landmarks. Each lanmark consists of the following:
|
||||
being the origin, and the smaller the value the closer the landmark is to
|
||||
the camera. The magnitude of `z` uses roughly the same scale as `x`.
|
||||
|
||||
Note: `z` is predicted only in full-body mode, and should be discarded when
|
||||
[upper_body_only](#upper_body_only) is `true`.
|
||||
|
||||
* `visibility`: A value in `[0.0, 1.0]` indicating the likelihood of the
|
||||
landmark being visible (present and not occluded) in the image.
|
||||
|
||||
@@ -184,7 +207,7 @@ install MediaPipe Python package, then learn more in the companion
|
||||
Supported configuration options:
|
||||
|
||||
* [static_image_mode](#static_image_mode)
|
||||
* [upper_body_only](#upper_body_only)
|
||||
* [model_complexity](#model_complexity)
|
||||
* [smooth_landmarks](#smooth_landmarks)
|
||||
* [min_detection_confidence](#min_detection_confidence)
|
||||
* [min_tracking_confidence](#min_tracking_confidence)
|
||||
@@ -197,7 +220,9 @@ mp_pose = mp.solutions.pose
|
||||
|
||||
# For static images:
|
||||
with mp_pose.Pose(
|
||||
static_image_mode=True, min_detection_confidence=0.5) as pose:
|
||||
static_image_mode=True,
|
||||
model_complexity=2,
|
||||
min_detection_confidence=0.5) as pose:
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
image_height, image_width, _ = image.shape
|
||||
@@ -213,8 +238,6 @@ with mp_pose.Pose(
|
||||
)
|
||||
# Draw pose landmarks on the image.
|
||||
annotated_image = image.copy()
|
||||
# Use mp_pose.UPPER_BODY_POSE_CONNECTIONS for drawing below when
|
||||
# upper_body_only is set to True.
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
@@ -258,7 +281,7 @@ and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
* [upperBodyOnly](#upper_body_only)
|
||||
* [modelComplexity](#model_complexity)
|
||||
* [smoothLandmarks](#smooth_landmarks)
|
||||
* [minDetectionConfidence](#min_detection_confidence)
|
||||
* [minTrackingConfidence](#min_tracking_confidence)
|
||||
@@ -305,7 +328,7 @@ const pose = new Pose({locateFile: (file) => {
|
||||
return `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`;
|
||||
}});
|
||||
pose.setOptions({
|
||||
upperBodyOnly: false,
|
||||
modelComplexity: 1,
|
||||
smoothLandmarks: true,
|
||||
minDetectionConfidence: 0.5,
|
||||
minTrackingConfidence: 0.5
|
||||
@@ -346,16 +369,6 @@ to visualize its associated subgraphs, please see
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/posetrackinggpu:PoseTrackingGpuApp`](http:/mediapipe/examples/ios/posetrackinggpu/BUILD)
|
||||
|
||||
#### Upper-body Only
|
||||
|
||||
* Graph:
|
||||
[`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt)
|
||||
* Android target:
|
||||
[(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1uKc6T7KSuA0Mlq2URi5YookHu0U3yoh_/view?usp=sharing)
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/upperbodyposetrackinggpu:upperbodyposetrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/upperbodyposetrackinggpu/BUILD)
|
||||
* iOS target:
|
||||
[`mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp`](http:/mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD)
|
||||
|
||||
### Desktop
|
||||
|
||||
Please first see general instructions for [desktop](../getting_started/cpp.md)
|
||||
@@ -374,134 +387,6 @@ on how to build MediaPipe examples.
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/pose_tracking:pose_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/pose_tracking/BUILD)
|
||||
|
||||
#### Upper-body Only
|
||||
|
||||
* Running on CPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_cpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_cpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
|
||||
* Running on GPU
|
||||
* Graph:
|
||||
[`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt)
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
|
||||
|
||||
## Pose Classification and Repetition Counting
|
||||
|
||||
One of the applications
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
can enable is fitness. More specifically - pose classification and repetition
|
||||
counting. In this section we'll provide basic guidance on building a custom pose
|
||||
classifier with the help of a
|
||||
[Colab](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
|
||||
and wrap it in a simple
|
||||
[fitness app](https://mediapipe.page.link/mlkit-pose-classification-demo-app)
|
||||
powered by [ML Kit](https://developers.google.com/ml-kit). Push-ups and squats
|
||||
are used for demonstration purposes as the most common exercises.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 4. Pose classification and repetition counting with MediaPipe Pose.* |
|
||||
|
||||
We picked the
|
||||
[k-nearest neighbors algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm)
|
||||
(k-NN) as the classifier. It's simple and easy to start with. The algorithm
|
||||
determines the object's class based on the closest samples in the training set.
|
||||
To build it, one needs to:
|
||||
|
||||
* Collect image samples of the target exercises and run pose prediction on
|
||||
them,
|
||||
* Convert obtained pose landmarks to a representation suitable for the k-NN
|
||||
classifier and form a training set,
|
||||
* Perform the classification itself followed by repetition counting.
|
||||
|
||||
### Training Set
|
||||
|
||||
To build a good classifier appropriate samples should be collected for the
|
||||
training set: about a few hundred samples for each terminal state of each
|
||||
exercise (e.g., "up" and "down" positions for push-ups). It's important that
|
||||
collected samples cover different camera angles, environment conditions, body
|
||||
shapes, and exercise variations.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 5. Two terminal states of push-ups.* |
|
||||
|
||||
To transform samples into a k-NN classifier training set, either
|
||||
[basic](https://drive.google.com/file/d/1z4IM8kG6ipHN6keadjD-F6vMiIIgViKK/view?usp=sharing)
|
||||
or
|
||||
[extended](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
|
||||
Colab could be used. They both use the
|
||||
[Python Solution API](#python-solution-api) to run the BlazePose models on given
|
||||
images and dump predicted pose landmarks to a CSV file. Additionally, the
|
||||
extended Colab provides useful tools to find outliers (e.g., wrongly predicted
|
||||
poses) and underrepresented classes (e.g., not covering all camera angles) by
|
||||
classifying each sample against the entire training set. After that, you'll be
|
||||
able to test the classifier on an arbitrary video right in the Colab.
|
||||
|
||||
### Classification
|
||||
|
||||
Code of the classifier is available both in the
|
||||
[extended](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
|
||||
Colab and in the
|
||||
[ML Kit demo app](https://mediapipe.page.link/mlkit-pose-classification-demo-app).
|
||||
Please refer to them for details of the approach described below.
|
||||
|
||||
The k-NN algorithm used for pose classification requires a feature vector
|
||||
representation of each sample and a metric to compute the distance between two
|
||||
such vectors to find the nearest pose samples to a target one.
|
||||
|
||||
To convert pose landmarks to a feature vector, we use pairwise distances between
|
||||
predefined lists of pose joints, such as distances between wrist and shoulder,
|
||||
ankle and hip, and two wrists. Since the algorithm relies on distances, all
|
||||
poses are normalized to have the same torso size and vertical torso orientation
|
||||
before the conversion.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 6. Main pairwise distances used for the pose feature vector.* |
|
||||
|
||||
To get a better classification result, k-NN search is invoked twice with
|
||||
different distance metrics:
|
||||
|
||||
* First, to filter out samples that are almost the same as the target one but
|
||||
have only a few different values in the feature vector (which means
|
||||
differently bent joints and thus other pose class), minimum per-coordinate
|
||||
distance is used as distance metric,
|
||||
* Then average per-coordinate distance is used to find the nearest pose
|
||||
cluster among those from the first search.
|
||||
|
||||
Finally, we apply
|
||||
[exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average)
|
||||
(EMA) smoothing to level any noise from pose prediction or classification. To do
|
||||
that, we search not only for the nearest pose cluster, but we calculate a
|
||||
probability for each of them and use it for smoothing over time.
|
||||
|
||||
### Repetition Counter
|
||||
|
||||
To count the repetitions, the algorithm monitors the probability of a target
|
||||
pose class. Let's take push-ups with its "up" and "down" terminal states:
|
||||
|
||||
* When the probability of the "down" pose class passes a certain threshold for
|
||||
the first time, the algorithm marks that the "down" pose class is entered.
|
||||
* Once the probability drops below the threshold, the algorithm marks that the
|
||||
"down" pose class has been exited and increases the counter.
|
||||
|
||||
To avoid cases when the probability fluctuates around the threshold (e.g., when
|
||||
the user pauses between "up" and "down" states) causing phantom counts, the
|
||||
threshold used to detect when the state is exited is actually slightly lower
|
||||
than the one used to detect when the state is entered. It creates an interval
|
||||
where the pose class and the counter can't be changed.
|
||||
|
||||
### Future Work
|
||||
|
||||
We are actively working on improving BlazePose GHUM 3D's Z prediction. It will
|
||||
allow us to use joint angles in the feature vectors, which are more natural and
|
||||
easier to configure (although distances can still be useful to detect touches
|
||||
between body parts) and to perform rotation normalization of poses and reduce
|
||||
the number of camera angles required for accurate k-NN classification.
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
@@ -512,5 +397,6 @@ the number of camera angles required for accurate k-NN classification.
|
||||
* [Models and model cards](./models.md#pose)
|
||||
* [Web demo](https://code.mediapipe.dev/codepen/pose)
|
||||
* [Python Colab](https://mediapipe.page.link/pose_py_colab)
|
||||
* [Pose Classification Colab (Basic)](https://mediapipe.page.link/pose_classification_basic)
|
||||
* [Pose Classification Colab (Extended)](https://mediapipe.page.link/pose_classification_extended)
|
||||
|
||||
[`mAP`]: https://cocodataset.org/#keypoints-eval
|
||||
[`[email protected]`]: https\://github.com/cbsudux/Human-Pose-Estimation-101
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
---
|
||||
layout: default
|
||||
title: Pose Classification
|
||||
parent: Pose
|
||||
grand_parent: Solutions
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
# Pose Classification
|
||||
{: .no_toc }
|
||||
|
||||
<details close markdown="block">
|
||||
<summary>
|
||||
Table of contents
|
||||
</summary>
|
||||
{: .text-delta }
|
||||
1. TOC
|
||||
{:toc}
|
||||
</details>
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
One of the applications
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
can enable is fitness. More specifically - pose classification and repetition
|
||||
counting. In this section we'll provide basic guidance on building a custom pose
|
||||
classifier with the help of [Colabs](#colabs) and wrap it in a simple fitness
|
||||
demo within
|
||||
[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
|
||||
Push-ups and squats are used for demonstration purposes as the most common
|
||||
exercises.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 1. Pose classification and repetition counting with MediaPipe Pose.* |
|
||||
|
||||
We picked the
|
||||
[k-nearest neighbors algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm)
|
||||
(k-NN) as the classifier. It's simple and easy to start with. The algorithm
|
||||
determines the object's class based on the closest samples in the training set.
|
||||
|
||||
**To build it, one needs to:**
|
||||
|
||||
1. Collect image samples of the target exercises and run pose prediction on
|
||||
them,
|
||||
2. Convert obtained pose landmarks to a representation suitable for the k-NN
|
||||
classifier and form a training set using these [Colabs](#colabs),
|
||||
3. Perform the classification itself followed by repetition counting (e.g., in
|
||||
the
|
||||
[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app)).
|
||||
|
||||
## Training Set
|
||||
|
||||
To build a good classifier appropriate samples should be collected for the
|
||||
training set: about a few hundred samples for each terminal state of each
|
||||
exercise (e.g., "up" and "down" positions for push-ups). It's important that
|
||||
collected samples cover different camera angles, environment conditions, body
|
||||
shapes, and exercise variations.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 2. Two terminal states of push-ups.* |
|
||||
|
||||
To transform samples into a k-NN classifier training set, both
|
||||
[`Pose Classification Colab (Basic)`] and
|
||||
[`Pose Classification Colab (Extended)`] could be used. They use the
|
||||
[Python Solution API](./pose.md#python-solution-api) to run the BlazePose models
|
||||
on given images and dump predicted pose landmarks to a CSV file. Additionally,
|
||||
the [`Pose Classification Colab (Extended)`] provides useful tools to find
|
||||
outliers (e.g., wrongly predicted poses) and underrepresented classes (e.g., not
|
||||
covering all camera angles) by classifying each sample against the entire
|
||||
training set. After that, you'll be able to test the classifier on an arbitrary
|
||||
video right in the Colab.
|
||||
|
||||
## Classification
|
||||
|
||||
Code of the classifier is available both in the
|
||||
[`Pose Classification Colab (Extended)`] and in the
|
||||
[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
|
||||
Please refer to them for details of the approach described below.
|
||||
|
||||
The k-NN algorithm used for pose classification requires a feature vector
|
||||
representation of each sample and a metric to compute the distance between two
|
||||
such vectors to find the nearest pose samples to a target one.
|
||||
|
||||
To convert pose landmarks to a feature vector, we use pairwise distances between
|
||||
predefined lists of pose joints, such as distances between wrist and shoulder,
|
||||
ankle and hip, and two wrists. Since the algorithm relies on distances, all
|
||||
poses are normalized to have the same torso size and vertical torso orientation
|
||||
before the conversion.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 3. Main pairwise distances used for the pose feature vector.* |
|
||||
|
||||
To get a better classification result, k-NN search is invoked twice with
|
||||
different distance metrics:
|
||||
|
||||
* First, to filter out samples that are almost the same as the target one but
|
||||
have only a few different values in the feature vector (which means
|
||||
differently bent joints and thus other pose class), minimum per-coordinate
|
||||
distance is used as distance metric,
|
||||
* Then average per-coordinate distance is used to find the nearest pose
|
||||
cluster among those from the first search.
|
||||
|
||||
Finally, we apply
|
||||
[exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average)
|
||||
(EMA) smoothing to level any noise from pose prediction or classification. To do
|
||||
that, we search not only for the nearest pose cluster, but we calculate a
|
||||
probability for each of them and use it for smoothing over time.
|
||||
|
||||
## Repetition Counting
|
||||
|
||||
To count the repetitions, the algorithm monitors the probability of a target
|
||||
pose class. Let's take push-ups with its "up" and "down" terminal states:
|
||||
|
||||
* When the probability of the "down" pose class passes a certain threshold for
|
||||
the first time, the algorithm marks that the "down" pose class is entered.
|
||||
* Once the probability drops below the threshold, the algorithm marks that the
|
||||
"down" pose class has been exited and increases the counter.
|
||||
|
||||
To avoid cases when the probability fluctuates around the threshold (e.g., when
|
||||
the user pauses between "up" and "down" states) causing phantom counts, the
|
||||
threshold used to detect when the state is exited is actually slightly lower
|
||||
than the one used to detect when the state is entered. It creates an interval
|
||||
where the pose class and the counter can't be changed.
|
||||
|
||||
## Future Work
|
||||
|
||||
We are actively working on improving
|
||||
[BlazePose GHUM 3D](./pose.md#pose-landmark-model-blazepose-ghum-3d)'s Z
|
||||
prediction. It will allow us to use joint angles in the feature vectors, which
|
||||
are more natural and easier to configure (although distances can still be useful
|
||||
to detect touches between body parts) and to perform rotation normalization of
|
||||
poses and reduce the number of camera angles required for accurate k-NN
|
||||
classification.
|
||||
|
||||
## Colabs
|
||||
|
||||
* [`Pose Classification Colab (Basic)`]
|
||||
* [`Pose Classification Colab (Extended)`]
|
||||
|
||||
[`Pose Classification Colab (Basic)`]: https://mediapipe.page.link/pose_classification_basic
|
||||
[`Pose Classification Colab (Extended)`]: https://mediapipe.page.link/pose_classification_extended
|
||||
@@ -28,7 +28,7 @@ has_toc: false
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
|
||||
@@ -41,6 +41,7 @@ profiler_config {
|
||||
trace_enabled: true
|
||||
enable_profiler: true
|
||||
trace_log_interval_count: 200
|
||||
trace_log_path: "/sdcard/Download/"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -64,7 +65,7 @@ MediaPipe will emit data into a pre-specified directory:
|
||||
|
||||
* On the desktop, this will be the `/tmp` directory.
|
||||
|
||||
* On Android, this will be the `/sdcard` directory.
|
||||
* On Android, this will be the external storage directory (e.g., `/storage/emulated/0/`).
|
||||
|
||||
* On iOS, this can be reached through XCode. Select "Window/Devices and
|
||||
Simulators" and select the "Devices" tab.
|
||||
@@ -103,7 +104,7 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
* Include the line below in your `AndroidManifest.xml` file.
|
||||
|
||||
```xml
|
||||
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
|
||||
<uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
|
||||
```
|
||||
|
||||
* Grant the permission either upon first app launch, or by going into
|
||||
@@ -130,8 +131,8 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
events to a trace log files at:
|
||||
|
||||
```bash
|
||||
/sdcard/mediapipe_trace_0.binarypb
|
||||
/sdcard/mediapipe_trace_1.binarypb
|
||||
/storage/emulated/0/Download/mediapipe_trace_0.binarypb
|
||||
/storage/emulated/0/Download/mediapipe_trace_1.binarypb
|
||||
```
|
||||
|
||||
After every 5 sec, writing shifts to a successive trace log file, such that
|
||||
@@ -139,10 +140,10 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
trace files have been written to the device using adb shell.
|
||||
|
||||
```bash
|
||||
adb shell "ls -la /sdcard/"
|
||||
adb shell "ls -la /storage/emulated/0/Download"
|
||||
```
|
||||
|
||||
On android, MediaPipe selects the external storage directory `/sdcard` for
|
||||
On android, MediaPipe selects the external storage (e.g., `/storage/emulated/0/`) for
|
||||
trace logs. This directory can be overridden using the setting
|
||||
`trace_log_path`, like:
|
||||
|
||||
@@ -150,7 +151,7 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
profiler_config {
|
||||
trace_enabled: true
|
||||
enable_profiler: true
|
||||
trace_log_path: "/sdcard/profiles/"
|
||||
trace_log_path: "/sdcard/Download/profiles/"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -161,7 +162,7 @@ we record ten intervals of half a second each. This can be overridden by adding
|
||||
|
||||
```bash
|
||||
# from your terminal
|
||||
adb pull /sdcard/mediapipe_trace_0.binarypb
|
||||
adb pull /storage/emulated/0/Download/mediapipe_trace_0.binarypb
|
||||
# if successful you should see something like
|
||||
# /sdcard/mediapipe_trace_0.binarypb: 1 file pulled. 0.1 MB/s (6766 bytes in 0.045s)
|
||||
```
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
"mediapipe/examples/ios/objectdetectiongpu/BUILD",
|
||||
"mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/posetrackinggpu/BUILD",
|
||||
"mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD",
|
||||
"mediapipe/framework/BUILD",
|
||||
"mediapipe/gpu/BUILD",
|
||||
"mediapipe/objc/BUILD",
|
||||
@@ -36,7 +35,6 @@
|
||||
"//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
|
||||
"//mediapipe/examples/ios/objectdetectiontrackinggpu:ObjectDetectionTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/posetrackinggpu:PoseTrackingGpuApp",
|
||||
"//mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp",
|
||||
"//mediapipe/objc:mediapipe_framework_ios"
|
||||
],
|
||||
"optionSet" : {
|
||||
@@ -105,7 +103,6 @@
|
||||
"mediapipe/examples/ios/objectdetectioncpu",
|
||||
"mediapipe/examples/ios/objectdetectiongpu",
|
||||
"mediapipe/examples/ios/posetrackinggpu",
|
||||
"mediapipe/examples/ios/upperbodyposetrackinggpu",
|
||||
"mediapipe/framework",
|
||||
"mediapipe/framework/deps",
|
||||
"mediapipe/framework/formats",
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
"mediapipe/examples/ios/objectdetectiongpu",
|
||||
"mediapipe/examples/ios/objectdetectiontrackinggpu",
|
||||
"mediapipe/examples/ios/posetrackinggpu",
|
||||
"mediapipe/examples/ios/upperbodyposetrackinggpu",
|
||||
"mediapipe/objc"
|
||||
],
|
||||
"projectName" : "Mediapipe",
|
||||
|
||||
@@ -128,7 +128,7 @@ cc_library(
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -147,7 +147,7 @@ cc_library(
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_audio_tools//audio/dsp/mfcc",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -168,7 +168,7 @@ cc_library(
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_audio_tools//audio/dsp:resampler",
|
||||
"@com_google_audio_tools//audio/dsp:resampler_q",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -208,7 +208,7 @@ cc_library(
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@com_google_audio_tools//audio/dsp/spectrogram",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -228,7 +228,7 @@ cc_library(
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -242,9 +242,9 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -261,7 +261,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -276,7 +276,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -296,7 +296,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@com_google_audio_tools//audio/dsp:number_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -314,7 +314,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -333,7 +333,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -352,6 +352,6 @@ cc_test(
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"//mediapipe/util:time_series_test_util",
|
||||
"@com_google_audio_tools//audio/dsp:signal_vector_util",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -12,10 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
@@ -25,7 +25,7 @@ namespace mediapipe {
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, TestWAV) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
@@ -34,7 +34,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -56,7 +56,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, Test48KWAV) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
@@ -65,7 +65,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -87,7 +87,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, TestMP3) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
@@ -96,7 +96,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
@@ -118,7 +118,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
|
||||
|
||||
TEST(AudioDecoderCalculatorTest, TestAAC) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "AudioDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_file_path"
|
||||
output_stream: "AUDIO:audio"
|
||||
@@ -127,7 +127,7 @@ TEST(AudioDecoderCalculatorTest, TestAAC) {
|
||||
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
|
||||
audio_stream { stream_index: 0 }
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
|
||||
file::JoinPath("./",
|
||||
|
||||
@@ -233,6 +233,22 @@ cc_test(
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "concatenate_vector_calculator_hdr",
|
||||
hdrs = ["concatenate_vector_calculator.h"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":concatenate_vector_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/api2:port",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "concatenate_vector_calculator",
|
||||
srcs = ["concatenate_vector_calculator.cc"],
|
||||
@@ -414,7 +430,7 @@ cc_library(
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -430,7 +446,7 @@ cc_library(
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -450,6 +466,35 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "non_zero_calculator",
|
||||
srcs = ["non_zero_calculator.cc"],
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "non_zero_calculator_test",
|
||||
size = "small",
|
||||
srcs = ["non_zero_calculator_test.cc"],
|
||||
deps = [
|
||||
":non_zero_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "mux_calculator_test",
|
||||
srcs = ["mux_calculator_test.cc"],
|
||||
@@ -651,6 +696,18 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "default_side_packet_calculator",
|
||||
srcs = ["default_side_packet_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "side_packet_to_stream_calculator",
|
||||
srcs = ["side_packet_to_stream_calculator.cc"],
|
||||
@@ -776,7 +833,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -793,7 +850,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1024,7 +1081,7 @@ cc_library(
|
||||
"//mediapipe/framework/tool:status_util",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@eigen_archive//:eigen",
|
||||
"@eigen_archive//:eigen3",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
@@ -70,7 +70,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
num_threads: 4
|
||||
input_stream: "ints"
|
||||
node {
|
||||
@@ -90,7 +90,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
|
||||
input_stream: "BATCH_END:timestamp"
|
||||
output_stream: "ITERABLE:ints_plus_one"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("ints_plus_one", &graph_config, &output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph_.StartRun({}));
|
||||
@@ -197,7 +197,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
|
||||
protected:
|
||||
void SetUp() override {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
num_threads: 4
|
||||
input_stream: "ints"
|
||||
input_stream: "force_ints_to_be_timestamp_bound_update"
|
||||
@@ -229,7 +229,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
|
||||
input_stream: "ints_plus_one"
|
||||
output_stream: "ints_plus_one_passed_through"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("ints_plus_one_passed_through", &graph_config,
|
||||
&output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config));
|
||||
@@ -338,7 +338,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
num_threads: 4
|
||||
input_stream: "ints"
|
||||
input_stream: "multiplier"
|
||||
@@ -362,7 +362,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
|
||||
input_stream: "BATCH_END:timestamp"
|
||||
output_stream: "ITERABLE:multiplied_ints"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("multiplied_ints", &graph_config, &output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph_.StartRun({}));
|
||||
|
||||
@@ -38,14 +38,14 @@ void AddInputVector(const std::vector<int>& input, int64 timestamp,
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
|
||||
std::vector<int> input = {};
|
||||
@@ -60,14 +60,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
|
||||
std::vector<int> input = {0, 1, 2, 3};
|
||||
@@ -85,14 +85,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
output_stream: "output_vector"
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
|
||||
{
|
||||
@@ -133,7 +133,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "TestClipUniqueIntPtrVectorSizeCalculator"
|
||||
@@ -143,7 +143,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("output_vector", &graph_config, &outputs);
|
||||
@@ -178,7 +178,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
|
||||
|
||||
TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "TestClipIntVectorSizeCalculator"
|
||||
input_stream: "input_vector"
|
||||
input_side_packet: "max_vec_size"
|
||||
@@ -186,7 +186,7 @@ TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
|
||||
options {
|
||||
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
// This should override the default of 1 set in the options.
|
||||
runner.MutableSidePackets()->Index(0) = Adopt(new int(2));
|
||||
|
||||
@@ -392,7 +392,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
input_stream: "in_3"
|
||||
@@ -403,7 +403,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
|
||||
input_stream: "in_3"
|
||||
output_stream: "out"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("out", &graph_config, &outputs);
|
||||
@@ -456,7 +456,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
node {
|
||||
@@ -465,7 +465,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
|
||||
input_stream: "in_2"
|
||||
output_stream: "out"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("out", &graph_config, &outputs);
|
||||
@@ -505,7 +505,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
* The test needs to send packets that own the data.
|
||||
*/
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "in_1"
|
||||
input_stream: "in_2"
|
||||
node {
|
||||
@@ -519,7 +519,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> outputs;
|
||||
tool::AddVectorSink("out", &graph_config, &outputs);
|
||||
|
||||
@@ -62,7 +62,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:0:int_packet"
|
||||
@@ -82,7 +82,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
@@ -111,7 +111,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:0:int_packet"
|
||||
@@ -131,7 +131,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
@@ -152,7 +152,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:int_packet"
|
||||
@@ -163,14 +163,14 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
EXPECT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:0:int_packet"
|
||||
@@ -181,7 +181,7 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
EXPECT_FALSE(graph.Initialize(graph_config).ok());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kOptionalValueTag[] = "OPTIONAL_VALUE";
|
||||
constexpr char kDefaultValueTag[] = "DEFAULT_VALUE";
|
||||
constexpr char kValueTag[] = "VALUE";
|
||||
|
||||
} // namespace
|
||||
|
||||
// Outputs side packet default value if optional value is not provided.
|
||||
//
|
||||
// This calculator utilizes the fact that MediaPipe automatically removes
|
||||
// optional side packets of the calculator configuration (i.e. OPTIONAL_VALUE).
|
||||
// And if it happens - returns default value, otherwise - returns optional
|
||||
// value.
|
||||
//
|
||||
// Input:
|
||||
// OPTIONAL_VALUE (optional) - AnyType (but same type as DEFAULT_VALUE)
|
||||
// Optional side packet value that is outputted by the calculator as is if
|
||||
// provided.
|
||||
//
|
||||
// DEFAULT_VALUE - AnyType
|
||||
// Default side pack value that is outputted by the calculator if
|
||||
// OPTIONAL_VALUE is not provided.
|
||||
//
|
||||
// Output:
|
||||
// VALUE - AnyType (but same type as DEFAULT_VALUE)
|
||||
// Either OPTIONAL_VALUE (if provided) or DEFAULT_VALUE (otherwise).
|
||||
//
|
||||
// Usage example:
|
||||
// node {
|
||||
// calculator: "DefaultSidePacketCalculator"
|
||||
// input_side_packet: "OPTIONAL_VALUE:segmentation_mask_enabled_optional"
|
||||
// input_side_packet: "DEFAULT_VALUE:segmentation_mask_enabled_default"
|
||||
// output_side_packet: "VALUE:segmentation_mask_enabled"
|
||||
// }
|
||||
class DefaultSidePacketCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(DefaultSidePacketCalculator);
|
||||
|
||||
absl::Status DefaultSidePacketCalculator::GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->InputSidePackets().HasTag(kDefaultValueTag))
|
||||
<< "Default value must be provided";
|
||||
cc->InputSidePackets().Tag(kDefaultValueTag).SetAny();
|
||||
|
||||
// Optional input side packet can be unspecified. In this case MediaPipe will
|
||||
// remove it from the calculator config.
|
||||
if (cc->InputSidePackets().HasTag(kOptionalValueTag)) {
|
||||
cc->InputSidePackets()
|
||||
.Tag(kOptionalValueTag)
|
||||
.SetSameAs(&cc->InputSidePackets().Tag(kDefaultValueTag))
|
||||
.Optional();
|
||||
}
|
||||
|
||||
RET_CHECK(cc->OutputSidePackets().HasTag(kValueTag));
|
||||
cc->OutputSidePackets().Tag(kValueTag).SetSameAs(
|
||||
&cc->InputSidePackets().Tag(kDefaultValueTag));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status DefaultSidePacketCalculator::Open(CalculatorContext* cc) {
|
||||
// If optional value is provided it is returned as the calculator output.
|
||||
if (cc->InputSidePackets().HasTag(kOptionalValueTag)) {
|
||||
auto& packet = cc->InputSidePackets().Tag(kOptionalValueTag);
|
||||
cc->OutputSidePackets().Tag(kValueTag).Set(packet);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// If no optional value
|
||||
auto& packet = cc->InputSidePackets().Tag(kDefaultValueTag);
|
||||
cc->OutputSidePackets().Tag(kValueTag).Set(packet);
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status DefaultSidePacketCalculator::Process(CalculatorContext* cc) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -27,7 +27,7 @@ namespace mediapipe {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
max_quantized_value: 2
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
@@ -51,7 +51,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
@@ -61,7 +61,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
min_quantized_value: 2
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
@@ -76,7 +76,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
@@ -86,7 +86,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::string empty_string;
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
@@ -101,7 +101,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
|
||||
TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "DequantizeByteArrayCalculator"
|
||||
input_stream: "ENCODED:encoded"
|
||||
output_stream: "FLOAT_VECTOR:float_vector"
|
||||
@@ -111,7 +111,7 @@ TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
|
||||
min_quantized_value: -2
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
|
||||
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
|
||||
|
||||
@@ -57,7 +57,7 @@ namespace mediapipe {
|
||||
//
|
||||
// The "ALLOW" stream indicates the transition between accepting frames and
|
||||
// dropping frames. "ALLOW = true" indicates the start of accepting frames
|
||||
// including the current timestamp, and "ALLOW = true" indicates the start of
|
||||
// including the current timestamp, and "ALLOW = false" indicates the start of
|
||||
// dropping frames including the current timestamp.
|
||||
//
|
||||
// FlowLimiterCalculator provides limited support for multiple input streams.
|
||||
|
||||
@@ -126,7 +126,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
|
||||
// Back-edge "finished" limits processing to one frame in-flight.
|
||||
// The LambdaCalculator is used to keep certain frames in flight.
|
||||
CalculatorGraphConfig InflightGraphConfig() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
node {
|
||||
calculator: 'FlowLimiterCalculator'
|
||||
@@ -143,7 +143,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
|
||||
input_stream: 'in_1_sampled'
|
||||
output_stream: 'out_1'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
protected:
|
||||
@@ -271,7 +271,7 @@ REGISTER_CALCULATOR(DropCalculator);
|
||||
class FlowLimiterCalculatorTest : public testing::Test {
|
||||
protected:
|
||||
CalculatorGraphConfig InflightGraphConfig() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
node {
|
||||
calculator: 'FlowLimiterCalculator'
|
||||
@@ -296,7 +296,7 @@ class FlowLimiterCalculatorTest : public testing::Test {
|
||||
input_stream: 'PACKET:out_1_sampled'
|
||||
output_stream: 'PACKET:out_1'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
// Parse an absl::Time from RFC3339 format.
|
||||
@@ -348,10 +348,10 @@ TEST_F(FlowLimiterCalculatorTest, FinishedTimestamps) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config = InflightGraphConfig();
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 1
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
@@ -419,11 +419,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedLost) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config = InflightGraphConfig();
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 1
|
||||
in_flight_timeout: 100000 # 100 ms
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
@@ -483,11 +483,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedDelayed) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config = InflightGraphConfig();
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 1
|
||||
in_flight_timeout: 100000 # 100 ms
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
@@ -548,7 +548,7 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
input_stream: 'in_2'
|
||||
node {
|
||||
@@ -576,13 +576,13 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
|
||||
input_stream: 'PACKET:out_1_sampled'
|
||||
output_stream: 'PACKET:out_1'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 1
|
||||
in_flight_timeout: 100000 # 100 ms
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
@@ -657,7 +657,7 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
|
||||
SetUpInputData();
|
||||
SetUpSimulationClock();
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
input_stream: 'in_2'
|
||||
node {
|
||||
@@ -685,13 +685,13 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
|
||||
input_stream: 'PACKET:out_1_sampled'
|
||||
output_stream: 'PACKET:out_1'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
|
||||
auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
|
||||
max_in_flight: 1
|
||||
max_in_queue: 0
|
||||
in_flight_timeout: 100000 # 100 ms
|
||||
)");
|
||||
)pb");
|
||||
std::map<std::string, Packet> side_packets = {
|
||||
{"limiter_options",
|
||||
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
|
||||
|
||||
@@ -64,13 +64,13 @@ const char kMatrixText2[] =
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "input_matrix"
|
||||
input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
input_side_packet: "MINUEND:side_matrix2"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
auto status = runner.Run();
|
||||
EXPECT_THAT(
|
||||
@@ -81,12 +81,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig) {
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
input_stream: "SUBTRAHEND:side_matrix2"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
auto status = runner.Run();
|
||||
EXPECT_THAT(status.message(), testing::HasSubstr("must be connected"));
|
||||
@@ -95,12 +95,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "MINUEND:input_matrix"
|
||||
input_side_packet: "SUBTRAHEND:side_matrix"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
Matrix* side_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText, side_matrix);
|
||||
@@ -124,12 +124,12 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
|
||||
|
||||
TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MatrixSubtractCalculator"
|
||||
input_stream: "SUBTRAHEND:input_matrix"
|
||||
input_side_packet: "MINUEND:side_matrix"
|
||||
output_stream: "output_matrix"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
Matrix* side_matrix = new Matrix();
|
||||
MatrixFromTextProto(kMatrixText, side_matrix);
|
||||
|
||||
@@ -26,31 +26,33 @@ namespace {
|
||||
|
||||
// Checks that the calculator fails if no input streams are provided.
|
||||
TEST(InvariantMergeInputStreamsCalculator, NoInputStreamsMustFail) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
output_stream: "merged_output"
|
||||
)"));
|
||||
)pb"));
|
||||
// Expect calculator to fail.
|
||||
ASSERT_FALSE(runner.Run().ok());
|
||||
}
|
||||
|
||||
// Checks that the calculator fails with an incorrect number of output streams.
|
||||
TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
|
||||
CalculatorRunner runner1(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
)"));
|
||||
CalculatorRunner runner1(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
)pb"));
|
||||
// Expect calculator to fail.
|
||||
EXPECT_FALSE(runner1.Run().ok());
|
||||
|
||||
CalculatorRunner runner2(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
output_stream: "output1"
|
||||
output_stream: "output2"
|
||||
)"));
|
||||
CalculatorRunner runner2(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
output_stream: "output1"
|
||||
output_stream: "output2"
|
||||
)pb"));
|
||||
// Expect calculator to fail.
|
||||
ASSERT_FALSE(runner2.Run().ok());
|
||||
}
|
||||
@@ -58,12 +60,12 @@ TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
|
||||
// Ensures two streams with differing types can be merged correctly.
|
||||
TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
|
||||
TestMergingTwoStreams) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
output_stream: "combined_output"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// input1: integers 10, 20, 30, occurring at times 10, 20, 30.
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
@@ -102,13 +104,13 @@ TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
|
||||
// Ensures three streams with differing types can be merged correctly.
|
||||
TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
|
||||
TestMergingThreeStreams) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "input1"
|
||||
input_stream: "input2"
|
||||
input_stream: "input3"
|
||||
output_stream: "combined_output"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
// input1: integer 30 occurring at time 30.
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
|
||||
@@ -31,7 +31,7 @@ namespace {
|
||||
|
||||
// Graph with default input stream handler, and the input selection is driven
|
||||
// by an input stream. All MuxCalculator inputs are present at each timestamp.
|
||||
constexpr char kTestGraphConfig1[] = R"proto(
|
||||
constexpr char kTestGraphConfig1[] = R"pb(
|
||||
input_stream: "input"
|
||||
output_stream: "test_output"
|
||||
node {
|
||||
@@ -60,12 +60,12 @@ constexpr char kTestGraphConfig1[] = R"proto(
|
||||
output_stream: "OUTPUT:test_output"
|
||||
input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
|
||||
}
|
||||
)proto";
|
||||
)pb";
|
||||
|
||||
// Graph with default input stream handler, and the input selection is driven
|
||||
// by an input side packet. All MuxCalculator inputs are present at each
|
||||
// timestamp.
|
||||
constexpr char kTestGraphConfig2[] = R"proto(
|
||||
constexpr char kTestGraphConfig2[] = R"pb(
|
||||
input_side_packet: "input_selector"
|
||||
input_stream: "input"
|
||||
output_stream: "test_output"
|
||||
@@ -93,12 +93,12 @@ constexpr char kTestGraphConfig2[] = R"proto(
|
||||
output_stream: "OUTPUT:test_output"
|
||||
input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
|
||||
}
|
||||
)proto";
|
||||
)pb";
|
||||
|
||||
// Graph with mux input stream handler, and the input selection is driven
|
||||
// by an input stream. Only one MuxCalculator input is present at each
|
||||
// timestamp.
|
||||
constexpr char kTestGraphConfig3[] = R"proto(
|
||||
constexpr char kTestGraphConfig3[] = R"pb(
|
||||
input_stream: "input"
|
||||
output_stream: "test_output"
|
||||
node {
|
||||
@@ -117,7 +117,7 @@ constexpr char kTestGraphConfig3[] = R"proto(
|
||||
input_stream: "SELECT:input_select"
|
||||
output_stream: "OUTPUT:test_output"
|
||||
}
|
||||
)proto";
|
||||
)pb";
|
||||
|
||||
constexpr char kOutputName[] = "test_output";
|
||||
constexpr char kInputName[] = "input";
|
||||
@@ -235,7 +235,7 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
|
||||
EXPECT_EQ(output, input_packets);
|
||||
}
|
||||
|
||||
constexpr char kDualInputGraphConfig[] = R"proto(
|
||||
constexpr char kDualInputGraphConfig[] = R"pb(
|
||||
input_stream: "input_0"
|
||||
input_stream: "input_1"
|
||||
input_stream: "input_select"
|
||||
@@ -247,7 +247,7 @@ constexpr char kDualInputGraphConfig[] = R"proto(
|
||||
input_stream: "SELECT:input_select"
|
||||
output_stream: "OUTPUT:test_output"
|
||||
}
|
||||
)proto";
|
||||
)pb";
|
||||
|
||||
TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
|
||||
CalculatorGraphConfig config =
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// A Calculator that returns 0 if INPUT is 0, and 1 otherwise.
|
||||
class NonZeroCalculator : public Node {
|
||||
public:
|
||||
static constexpr Input<int>::SideFallback kIn{"INPUT"};
|
||||
static constexpr Output<int>::Optional kOut{"OUTPUT"};
|
||||
static constexpr Output<bool>::Optional kBooleanOut{"OUTPUT_BOOL"};
|
||||
|
||||
MEDIAPIPE_NODE_CONTRACT(kIn, kOut, kBooleanOut);
|
||||
|
||||
absl::Status UpdateContract(CalculatorContract* cc) {
|
||||
RET_CHECK(kOut(cc).IsConnected() || kBooleanOut(cc).IsConnected())
|
||||
<< "At least one output stream is expected.";
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
if (!kIn(cc).IsEmpty()) {
|
||||
bool isNonZero = *kIn(cc) != 0;
|
||||
if (kOut(cc).IsConnected()) {
|
||||
kOut(cc).Send(std::make_unique<int>(isNonZero ? 1 : 0));
|
||||
}
|
||||
if (kBooleanOut(cc).IsConnected()) {
|
||||
kBooleanOut(cc).Send(std::make_unique<bool>(isNonZero));
|
||||
}
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
MEDIAPIPE_REGISTER_NODE(NonZeroCalculator);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,93 @@
|
||||
// Copyright 2021 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/framework/tool/validate_type.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class NonZeroCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
NonZeroCalculatorTest()
|
||||
: runner_(
|
||||
R"pb(
|
||||
calculator: "NonZeroCalculator"
|
||||
input_stream: "INPUT:input"
|
||||
output_stream: "OUTPUT:output"
|
||||
output_stream: "OUTPUT_BOOL:output_bool"
|
||||
)pb") {}
|
||||
|
||||
void SetInput(const std::vector<int>& inputs) {
|
||||
int timestamp = 0;
|
||||
for (const auto input : inputs) {
|
||||
runner_.MutableInputs()
|
||||
->Get("INPUT", 0)
|
||||
.packets.push_back(MakePacket<int>(input).At(Timestamp(timestamp++)));
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<int> GetOutput() {
|
||||
std::vector<int> result;
|
||||
for (const auto output : runner_.Outputs().Get("OUTPUT", 0).packets) {
|
||||
result.push_back(output.Get<int>());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<bool> GetOutputBool() {
|
||||
std::vector<bool> result;
|
||||
for (const auto output : runner_.Outputs().Get("OUTPUT_BOOL", 0).packets) {
|
||||
result.push_back(output.Get<bool>());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
CalculatorRunner runner_;
|
||||
};
|
||||
|
||||
TEST_F(NonZeroCalculatorTest, ProducesZeroOutputForZeroInput) {
|
||||
SetInput({0});
|
||||
|
||||
MP_ASSERT_OK(runner_.Run());
|
||||
|
||||
EXPECT_THAT(GetOutput(), ::testing::ElementsAre(0));
|
||||
EXPECT_THAT(GetOutputBool(), ::testing::ElementsAre(false));
|
||||
}
|
||||
|
||||
TEST_F(NonZeroCalculatorTest, ProducesNonZeroOutputForNonZeroInput) {
|
||||
SetInput({1, 2, 3, -4, 5});
|
||||
|
||||
MP_ASSERT_OK(runner_.Run());
|
||||
|
||||
EXPECT_THAT(GetOutput(), ::testing::ElementsAre(1, 1, 1, 1, 1));
|
||||
EXPECT_THAT(GetOutputBool(),
|
||||
::testing::ElementsAre(true, true, true, true, true));
|
||||
}
|
||||
|
||||
TEST_F(NonZeroCalculatorTest, SwitchesBetweenNonZeroAndZeroOutput) {
|
||||
SetInput({1, 0, 3, 0, 5});
|
||||
MP_ASSERT_OK(runner_.Run());
|
||||
EXPECT_THAT(GetOutput(), ::testing::ElementsAre(1, 0, 1, 0, 1));
|
||||
EXPECT_THAT(GetOutputBool(),
|
||||
::testing::ElementsAre(true, false, true, false, true));
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -40,7 +40,7 @@ MATCHER_P2(BoolPacket, value, timestamp, "") {
|
||||
TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
std::vector<Packet> output_packets;
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'allow'
|
||||
input_stream: 'value'
|
||||
node {
|
||||
@@ -54,7 +54,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
input_stream: 'PACKET:gated_value'
|
||||
output_stream: 'PRESENCE:presence'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("presence", &graph_config, &output_packets);
|
||||
|
||||
CalculatorGraph graph;
|
||||
|
||||
@@ -87,7 +87,6 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
|
||||
|
||||
flush_last_packet_ = resampler_options.flush_last_packet();
|
||||
jitter_ = resampler_options.jitter();
|
||||
jitter_with_reflection_ = resampler_options.jitter_with_reflection();
|
||||
|
||||
input_data_id_ = cc->Inputs().GetId("DATA", 0);
|
||||
if (!input_data_id_.IsValid()) {
|
||||
@@ -98,11 +97,7 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
|
||||
output_data_id_ = cc->Outputs().GetId("", 0);
|
||||
}
|
||||
|
||||
period_count_ = 0;
|
||||
frame_rate_ = resampler_options.frame_rate();
|
||||
base_timestamp_ = resampler_options.has_base_timestamp()
|
||||
? Timestamp(resampler_options.base_timestamp())
|
||||
: Timestamp::Unset();
|
||||
start_time_ = resampler_options.has_start_time()
|
||||
? Timestamp(resampler_options.start_time())
|
||||
: Timestamp::Min();
|
||||
@@ -141,30 +136,9 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
|
||||
}
|
||||
}
|
||||
|
||||
if (jitter_ != 0.0) {
|
||||
if (resampler_options.output_header() !=
|
||||
PacketResamplerCalculatorOptions::NONE) {
|
||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||
"the actual value.";
|
||||
}
|
||||
if (flush_last_packet_) {
|
||||
flush_last_packet_ = false;
|
||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::Status(
|
||||
absl::StatusCode::kInvalidArgument,
|
||||
"SecureRandom is not available. With \"jitter\" specified, "
|
||||
"PacketResamplerCalculator processing cannot proceed.");
|
||||
}
|
||||
packet_reservoir_random_ = CreateSecureRandom(seed);
|
||||
}
|
||||
packet_reservoir_ =
|
||||
std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
|
||||
return absl::OkStatus();
|
||||
strategy_ = GetSamplingStrategy(resampler_options);
|
||||
|
||||
return strategy_->Open(cc);
|
||||
}
|
||||
|
||||
absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
|
||||
@@ -177,171 +151,13 @@ absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
}
|
||||
if (jitter_ != 0.0 && random_ != nullptr) {
|
||||
// Packet reservior is used to make sure there's an output for every period,
|
||||
// e.g. partial period at the end of the stream.
|
||||
if (packet_reservoir_->IsEnabled() &&
|
||||
(first_timestamp_ == Timestamp::Unset() ||
|
||||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
|
||||
auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
|
||||
packet_reservoir_->AddSample(curr_packet);
|
||||
}
|
||||
MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
|
||||
} else {
|
||||
MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
|
||||
|
||||
if (absl::Status status = strategy_->Process(cc); !status.ok()) {
|
||||
return status; // Avoid MP_RETURN_IF_ERROR macro for external release.
|
||||
}
|
||||
|
||||
last_packet_ = cc->Inputs().Get(input_data_id_).Value();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
|
||||
next_output_timestamp_min_ = first_timestamp_;
|
||||
if (jitter_with_reflection_) {
|
||||
next_output_timestamp_ =
|
||||
first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
|
||||
return;
|
||||
}
|
||||
next_output_timestamp_ =
|
||||
first_timestamp_ + frame_time_usec_ * random_->RandFloat();
|
||||
}
|
||||
|
||||
void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
|
||||
packet_reservoir_->Clear();
|
||||
if (jitter_with_reflection_) {
|
||||
next_output_timestamp_min_ += frame_time_usec_;
|
||||
Timestamp next_output_timestamp_max_ =
|
||||
next_output_timestamp_min_ + frame_time_usec_;
|
||||
|
||||
next_output_timestamp_ += frame_time_usec_ +
|
||||
random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
|
||||
jitter_usec_;
|
||||
next_output_timestamp_ = Timestamp(ReflectBetween(
|
||||
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
||||
next_output_timestamp_max_.Value()));
|
||||
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
|
||||
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
|
||||
return;
|
||||
}
|
||||
packet_reservoir_->Disable();
|
||||
next_output_timestamp_ +=
|
||||
frame_time_usec_ *
|
||||
((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
|
||||
}
|
||||
|
||||
absl::Status PacketResamplerCalculator::ProcessWithJitter(
|
||||
CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
RET_CHECK_NE(jitter_, 0.0);
|
||||
|
||||
if (first_timestamp_ == Timestamp::Unset()) {
|
||||
first_timestamp_ = cc->InputTimestamp();
|
||||
InitializeNextOutputTimestampWithJitter();
|
||||
if (first_timestamp_ == next_output_timestamp_) {
|
||||
OutputWithinLimits(
|
||||
cc,
|
||||
cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
if (frame_time_usec_ <
|
||||
(cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
|
||||
LOG_FIRST_N(WARNING, 2)
|
||||
<< "Adding jitter is not very useful when upsampling.";
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const int64 last_diff =
|
||||
(next_output_timestamp_ - last_packet_.Timestamp()).Value();
|
||||
RET_CHECK_GT(last_diff, 0);
|
||||
const int64 curr_diff =
|
||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||
if (curr_diff > 0) {
|
||||
break;
|
||||
}
|
||||
OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
|
||||
? last_packet_
|
||||
: cc->Inputs().Get(input_data_id_).Value())
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
// From now on every time a packet is emitted the timestamp of the next
|
||||
// packet becomes known; that timestamp is stored in next_output_timestamp_.
|
||||
// The only exception to this rule is the packet emitted from Close() which
|
||||
// can only happen when jitter_with_reflection is enabled but in this case
|
||||
// next_output_timestamp_min_ is a non-decreasing lower bound of any
|
||||
// subsequent packet.
|
||||
const Timestamp timestamp_bound = jitter_with_reflection_
|
||||
? next_output_timestamp_min_
|
||||
: next_output_timestamp_;
|
||||
cc->Outputs().Get(output_data_id_).SetNextTimestampBound(timestamp_bound);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status PacketResamplerCalculator::ProcessWithoutJitter(
|
||||
CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
RET_CHECK_EQ(jitter_, 0.0);
|
||||
|
||||
if (first_timestamp_ == Timestamp::Unset()) {
|
||||
// This is the first packet, initialize the first_timestamp_.
|
||||
if (base_timestamp_ == Timestamp::Unset()) {
|
||||
// Initialize first_timestamp_ with exactly the first packet timestamp.
|
||||
first_timestamp_ = cc->InputTimestamp();
|
||||
} else {
|
||||
// Initialize first_timestamp_ with the first packet timestamp
|
||||
// aligned to the base_timestamp_.
|
||||
int64 first_index = MathUtil::SafeRound<int64, double>(
|
||||
(cc->InputTimestamp() - base_timestamp_).Seconds() * frame_rate_);
|
||||
first_timestamp_ =
|
||||
base_timestamp_ + TimestampDiffFromSeconds(first_index / frame_rate_);
|
||||
}
|
||||
if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
|
||||
cc->Outputs()
|
||||
.Tag("VIDEO_HEADER")
|
||||
.Add(new VideoHeader(video_header_), Timestamp::PreStream());
|
||||
}
|
||||
}
|
||||
const Timestamp received_timestamp = cc->InputTimestamp();
|
||||
const int64 received_timestamp_idx =
|
||||
TimestampToPeriodIndex(received_timestamp);
|
||||
// Only consider the received packet if it belongs to the current period
|
||||
// (== period_count_) or to a newer one (> period_count_).
|
||||
if (received_timestamp_idx >= period_count_) {
|
||||
// Fill the empty periods until we are in the same index as the received
|
||||
// packet.
|
||||
while (received_timestamp_idx > period_count_) {
|
||||
OutputWithinLimits(
|
||||
cc, last_packet_.At(PeriodIndexToTimestamp(period_count_)));
|
||||
++period_count_;
|
||||
}
|
||||
// Now, if the received packet has a timestamp larger than the middle of
|
||||
// the current period, we can send a packet without waiting. We send the
|
||||
// one closer to the middle.
|
||||
Timestamp target_timestamp = PeriodIndexToTimestamp(period_count_);
|
||||
if (received_timestamp >= target_timestamp) {
|
||||
bool have_last_packet = (last_packet_.Timestamp() != Timestamp::Unset());
|
||||
bool send_current =
|
||||
!have_last_packet || (received_timestamp - target_timestamp <=
|
||||
target_timestamp - last_packet_.Timestamp());
|
||||
if (send_current) {
|
||||
OutputWithinLimits(
|
||||
cc, cc->Inputs().Get(input_data_id_).Value().At(target_timestamp));
|
||||
} else {
|
||||
OutputWithinLimits(cc, last_packet_.At(target_timestamp));
|
||||
}
|
||||
++period_count_;
|
||||
}
|
||||
// TODO: Add a mechanism to the framework to allow these packets
|
||||
// to be output earlier (without waiting for a much later packet to
|
||||
// arrive)
|
||||
|
||||
// Update the bound for the next packet.
|
||||
cc->Outputs()
|
||||
.Get(output_data_id_)
|
||||
.SetNextTimestampBound(PeriodIndexToTimestamp(period_count_));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -349,17 +165,34 @@ absl::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
|
||||
if (!cc->GraphStatus().ok()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
// Emit the last packet received if we have at least one packet, but
|
||||
// haven't sent anything for its period.
|
||||
if (first_timestamp_ != Timestamp::Unset() && flush_last_packet_ &&
|
||||
TimestampToPeriodIndex(last_packet_.Timestamp()) == period_count_) {
|
||||
OutputWithinLimits(cc,
|
||||
last_packet_.At(PeriodIndexToTimestamp(period_count_)));
|
||||
|
||||
return strategy_->Close(cc);
|
||||
}
|
||||
|
||||
std::unique_ptr<PacketResamplerStrategy>
|
||||
PacketResamplerCalculator::GetSamplingStrategy(
|
||||
const PacketResamplerCalculatorOptions& options) {
|
||||
if (options.reproducible_sampling()) {
|
||||
if (!options.jitter_with_reflection()) {
|
||||
LOG(WARNING)
|
||||
<< "reproducible_sampling enabled w/ jitter_with_reflection "
|
||||
"disabled. "
|
||||
<< "reproducible_sampling always uses jitter with reflection, "
|
||||
<< "Ignoring jitter_with_reflection setting.";
|
||||
}
|
||||
return absl::make_unique<ReproducibleJitterWithReflectionStrategy>(this);
|
||||
}
|
||||
if (!packet_reservoir_->IsEmpty()) {
|
||||
OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
||||
|
||||
if (options.jitter() == 0) {
|
||||
return absl::make_unique<NoJitterStrategy>(this);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
|
||||
if (options.jitter_with_reflection()) {
|
||||
return absl::make_unique<LegacyJitterWithReflectionStrategy>(this);
|
||||
}
|
||||
|
||||
// With jitter and no reflection.
|
||||
return absl::make_unique<JitterWithoutReflectionStrategy>(this);
|
||||
}
|
||||
|
||||
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const {
|
||||
@@ -385,4 +218,479 @@ void PacketResamplerCalculator::OutputWithinLimits(CalculatorContext* cc,
|
||||
}
|
||||
}
|
||||
|
||||
absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
|
||||
const auto resampler_options =
|
||||
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
|
||||
cc->InputSidePackets(), "OPTIONS");
|
||||
|
||||
if (resampler_options.output_header() !=
|
||||
PacketResamplerCalculatorOptions::NONE) {
|
||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||
"the actual value.";
|
||||
}
|
||||
|
||||
if (calculator_->flush_last_packet_) {
|
||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::InvalidArgumentError(
|
||||
"SecureRandom is not available. With \"jitter\" specified, "
|
||||
"PacketResamplerCalculator processing cannot proceed.");
|
||||
}
|
||||
|
||||
packet_reservoir_random_ = CreateSecureRandom(seed);
|
||||
packet_reservoir_ =
|
||||
std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
|
||||
if (!packet_reservoir_->IsEmpty()) {
|
||||
LOG(INFO) << "Emitting pack from reservoir.";
|
||||
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status LegacyJitterWithReflectionStrategy::Process(
|
||||
CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
|
||||
if (packet_reservoir_->IsEnabled() &&
|
||||
(first_timestamp_ == Timestamp::Unset() ||
|
||||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
|
||||
auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
|
||||
packet_reservoir_->AddSample(curr_packet);
|
||||
}
|
||||
|
||||
if (first_timestamp_ == Timestamp::Unset()) {
|
||||
first_timestamp_ = cc->InputTimestamp();
|
||||
InitializeNextOutputTimestampWithJitter();
|
||||
if (first_timestamp_ == next_output_timestamp_) {
|
||||
calculator_->OutputWithinLimits(cc, cc->Inputs()
|
||||
.Get(calculator_->input_data_id_)
|
||||
.Value()
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
if (calculator_->frame_time_usec_ <
|
||||
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
||||
LOG_FIRST_N(WARNING, 2)
|
||||
<< "Adding jitter is not very useful when upsampling.";
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const int64 last_diff =
|
||||
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
|
||||
.Value();
|
||||
RET_CHECK_GT(last_diff, 0);
|
||||
const int64 curr_diff =
|
||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||
if (curr_diff > 0) {
|
||||
break;
|
||||
}
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, (std::abs(curr_diff) > last_diff
|
||||
? calculator_->last_packet_
|
||||
: cc->Inputs().Get(calculator_->input_data_id_).Value())
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestampWithJitter();
|
||||
// From now on every time a packet is emitted the timestamp of the next
|
||||
// packet becomes known; that timestamp is stored in next_output_timestamp_.
|
||||
// The only exception to this rule is the packet emitted from Close() which
|
||||
// can only happen when jitter_with_reflection is enabled but in this case
|
||||
// next_output_timestamp_min_ is a non-decreasing lower bound of any
|
||||
// subsequent packet.
|
||||
const Timestamp timestamp_bound = next_output_timestamp_min_;
|
||||
cc->Outputs()
|
||||
.Get(calculator_->output_data_id_)
|
||||
.SetNextTimestampBound(timestamp_bound);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void LegacyJitterWithReflectionStrategy::
|
||||
InitializeNextOutputTimestampWithJitter() {
|
||||
next_output_timestamp_min_ = first_timestamp_;
|
||||
next_output_timestamp_ =
|
||||
first_timestamp_ +
|
||||
random_->UnbiasedUniform64(calculator_->frame_time_usec_);
|
||||
}
|
||||
|
||||
void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
|
||||
packet_reservoir_->Clear();
|
||||
next_output_timestamp_min_ += calculator_->frame_time_usec_;
|
||||
Timestamp next_output_timestamp_max_ =
|
||||
next_output_timestamp_min_ + calculator_->frame_time_usec_;
|
||||
|
||||
next_output_timestamp_ +=
|
||||
calculator_->frame_time_usec_ +
|
||||
random_->UnbiasedUniform64(2 * calculator_->jitter_usec_ + 1) -
|
||||
calculator_->jitter_usec_;
|
||||
next_output_timestamp_ = Timestamp(ReflectBetween(
|
||||
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
||||
next_output_timestamp_max_.Value()));
|
||||
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
|
||||
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
|
||||
}
|
||||
|
||||
absl::Status ReproducibleJitterWithReflectionStrategy::Open(
|
||||
CalculatorContext* cc) {
|
||||
const auto resampler_options =
|
||||
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
|
||||
cc->InputSidePackets(), "OPTIONS");
|
||||
|
||||
if (resampler_options.output_header() !=
|
||||
PacketResamplerCalculatorOptions::NONE) {
|
||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||
"the actual value.";
|
||||
}
|
||||
|
||||
if (calculator_->flush_last_packet_) {
|
||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::InvalidArgumentError(
|
||||
"SecureRandom is not available. With \"jitter\" specified, "
|
||||
"PacketResamplerCalculator processing cannot proceed.");
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status ReproducibleJitterWithReflectionStrategy::Close(
|
||||
CalculatorContext* cc) {
|
||||
// If last packet is non-empty and a packet hasn't been emitted for this
|
||||
// period, emit the last packet.
|
||||
if (!calculator_->last_packet_.IsEmpty() && !packet_emitted_this_period_) {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, calculator_->last_packet_.At(next_output_timestamp_));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status ReproducibleJitterWithReflectionStrategy::Process(
|
||||
CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
|
||||
Packet current_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
|
||||
|
||||
if (calculator_->last_packet_.IsEmpty()) {
|
||||
// last_packet is empty, this is the first packet of the stream.
|
||||
|
||||
InitializeNextOutputTimestamp(current_packet.Timestamp());
|
||||
|
||||
// If next_output_timestamp_ happens to fall before current_packet, emit
|
||||
// current packet. Only a single packet can be emitted at the beginning
|
||||
// of the stream.
|
||||
if (next_output_timestamp_ < current_packet.Timestamp()) {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, current_packet.At(next_output_timestamp_));
|
||||
packet_emitted_this_period_ = true;
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// Last packet is set, so we are mid-stream.
|
||||
if (calculator_->frame_time_usec_ <
|
||||
(current_packet.Timestamp() - calculator_->last_packet_.Timestamp())
|
||||
.Value()) {
|
||||
// Note, if the stream is upsampling, this could lead to the same packet
|
||||
// being emitted twice. Upsampling and jitter doesn't make much sense
|
||||
// but does technically work.
|
||||
LOG_FIRST_N(WARNING, 2)
|
||||
<< "Adding jitter is not very useful when upsampling.";
|
||||
}
|
||||
|
||||
// Since we may be upsampling, we need to iteratively advance the
|
||||
// next_output_timestamp_ one period at a time until it reaches the period
|
||||
// current_packet is in. During this process, last_packet and/or
|
||||
// current_packet may be repeatly emitted.
|
||||
|
||||
UpdateNextOutputTimestamp(current_packet.Timestamp());
|
||||
|
||||
while (!packet_emitted_this_period_ &&
|
||||
next_output_timestamp_ <= current_packet.Timestamp()) {
|
||||
// last_packet < next_output_timestamp_ <= current_packet,
|
||||
// so emit the closest packet.
|
||||
Packet packet_to_emit =
|
||||
current_packet.Timestamp() - next_output_timestamp_ <
|
||||
next_output_timestamp_ - calculator_->last_packet_.Timestamp()
|
||||
? current_packet
|
||||
: calculator_->last_packet_;
|
||||
calculator_->OutputWithinLimits(cc,
|
||||
packet_to_emit.At(next_output_timestamp_));
|
||||
|
||||
packet_emitted_this_period_ = true;
|
||||
|
||||
// If we are upsampling, packet_emitted_this_period_ can be reset by
|
||||
// the following UpdateNext and the loop will iterate.
|
||||
UpdateNextOutputTimestamp(current_packet.Timestamp());
|
||||
}
|
||||
|
||||
// Set the bounds on the output stream. Note, if we emitted a packet
|
||||
// above, it will already be set at next_output_timestamp_ + 1, in which
|
||||
// case we have to skip setting it.
|
||||
if (cc->Outputs().Get(calculator_->output_data_id_).NextTimestampBound() <
|
||||
next_output_timestamp_) {
|
||||
cc->Outputs()
|
||||
.Get(calculator_->output_data_id_)
|
||||
.SetNextTimestampBound(next_output_timestamp_);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void ReproducibleJitterWithReflectionStrategy::InitializeNextOutputTimestamp(
|
||||
Timestamp current_timestamp) {
|
||||
if (next_output_timestamp_min_ != Timestamp::Unset()) {
|
||||
return;
|
||||
}
|
||||
|
||||
next_output_timestamp_min_ = Timestamp(0);
|
||||
next_output_timestamp_ =
|
||||
Timestamp(GetNextRandom(calculator_->frame_time_usec_));
|
||||
|
||||
// While the current timestamp is ahead of the max (i.e. min + frame_time),
|
||||
// fast-forward.
|
||||
while (current_timestamp >=
|
||||
next_output_timestamp_min_ + calculator_->frame_time_usec_) {
|
||||
packet_emitted_this_period_ = true; // Force update...
|
||||
UpdateNextOutputTimestamp(current_timestamp);
|
||||
}
|
||||
}
|
||||
|
||||
void ReproducibleJitterWithReflectionStrategy::UpdateNextOutputTimestamp(
|
||||
Timestamp current_timestamp) {
|
||||
if (packet_emitted_this_period_ &&
|
||||
current_timestamp >=
|
||||
next_output_timestamp_min_ + calculator_->frame_time_usec_) {
|
||||
next_output_timestamp_min_ += calculator_->frame_time_usec_;
|
||||
Timestamp next_output_timestamp_max_ =
|
||||
next_output_timestamp_min_ + calculator_->frame_time_usec_;
|
||||
|
||||
next_output_timestamp_ += calculator_->frame_time_usec_ +
|
||||
GetNextRandom(2 * calculator_->jitter_usec_ + 1) -
|
||||
calculator_->jitter_usec_;
|
||||
next_output_timestamp_ = Timestamp(ReflectBetween(
|
||||
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
||||
next_output_timestamp_max_.Value()));
|
||||
|
||||
packet_emitted_this_period_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
|
||||
const auto resampler_options =
|
||||
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
|
||||
cc->InputSidePackets(), "OPTIONS");
|
||||
|
||||
if (resampler_options.output_header() !=
|
||||
PacketResamplerCalculatorOptions::NONE) {
|
||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||
"the actual value.";
|
||||
}
|
||||
|
||||
if (calculator_->flush_last_packet_) {
|
||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||
"ignored, because we are adding jitter.";
|
||||
}
|
||||
|
||||
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
|
||||
random_ = CreateSecureRandom(seed);
|
||||
if (random_ == nullptr) {
|
||||
return absl::InvalidArgumentError(
|
||||
"SecureRandom is not available. With \"jitter\" specified, "
|
||||
"PacketResamplerCalculator processing cannot proceed.");
|
||||
}
|
||||
|
||||
packet_reservoir_random_ = CreateSecureRandom(seed);
|
||||
packet_reservoir_ =
|
||||
absl::make_unique<PacketReservoir>(packet_reservoir_random_.get());
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status JitterWithoutReflectionStrategy::Close(CalculatorContext* cc) {
|
||||
if (!packet_reservoir_->IsEmpty()) {
|
||||
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
|
||||
// Packet reservior is used to make sure there's an output for every period,
|
||||
// e.g. partial period at the end of the stream.
|
||||
if (packet_reservoir_->IsEnabled() &&
|
||||
(calculator_->first_timestamp_ == Timestamp::Unset() ||
|
||||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
|
||||
auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
|
||||
packet_reservoir_->AddSample(curr_packet);
|
||||
}
|
||||
|
||||
if (calculator_->first_timestamp_ == Timestamp::Unset()) {
|
||||
calculator_->first_timestamp_ = cc->InputTimestamp();
|
||||
InitializeNextOutputTimestamp();
|
||||
if (calculator_->first_timestamp_ == next_output_timestamp_) {
|
||||
calculator_->OutputWithinLimits(cc, cc->Inputs()
|
||||
.Get(calculator_->input_data_id_)
|
||||
.Value()
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestamp();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
if (calculator_->frame_time_usec_ <
|
||||
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
||||
LOG_FIRST_N(WARNING, 2)
|
||||
<< "Adding jitter is not very useful when upsampling.";
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const int64 last_diff =
|
||||
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
|
||||
.Value();
|
||||
RET_CHECK_GT(last_diff, 0);
|
||||
const int64 curr_diff =
|
||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||
if (curr_diff > 0) {
|
||||
break;
|
||||
}
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, (std::abs(curr_diff) > last_diff
|
||||
? calculator_->last_packet_
|
||||
: cc->Inputs().Get(calculator_->input_data_id_).Value())
|
||||
.At(next_output_timestamp_));
|
||||
UpdateNextOutputTimestamp();
|
||||
cc->Outputs()
|
||||
.Get(calculator_->output_data_id_)
|
||||
.SetNextTimestampBound(next_output_timestamp_);
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void JitterWithoutReflectionStrategy::InitializeNextOutputTimestamp() {
|
||||
next_output_timestamp_min_ = calculator_->first_timestamp_;
|
||||
next_output_timestamp_ = calculator_->first_timestamp_ +
|
||||
calculator_->frame_time_usec_ * random_->RandFloat();
|
||||
}
|
||||
|
||||
void JitterWithoutReflectionStrategy::UpdateNextOutputTimestamp() {
|
||||
packet_reservoir_->Clear();
|
||||
packet_reservoir_->Disable();
|
||||
next_output_timestamp_ += calculator_->frame_time_usec_ *
|
||||
((1.0 - calculator_->jitter_) +
|
||||
2.0 * calculator_->jitter_ * random_->RandFloat());
|
||||
}
|
||||
|
||||
absl::Status NoJitterStrategy::Open(CalculatorContext* cc) {
|
||||
const auto resampler_options =
|
||||
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
|
||||
cc->InputSidePackets(), "OPTIONS");
|
||||
base_timestamp_ = resampler_options.has_base_timestamp()
|
||||
? Timestamp(resampler_options.base_timestamp())
|
||||
: Timestamp::Unset();
|
||||
|
||||
period_count_ = 0;
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status NoJitterStrategy::Close(CalculatorContext* cc) {
|
||||
// Emit the last packet received if we have at least one packet, but
|
||||
// haven't sent anything for its period.
|
||||
if (calculator_->first_timestamp_ != Timestamp::Unset() &&
|
||||
calculator_->flush_last_packet_ &&
|
||||
calculator_->TimestampToPeriodIndex(
|
||||
calculator_->last_packet_.Timestamp()) == period_count_) {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, calculator_->last_packet_.At(
|
||||
calculator_->PeriodIndexToTimestamp(period_count_)));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
|
||||
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
|
||||
|
||||
if (calculator_->first_timestamp_ == Timestamp::Unset()) {
|
||||
// This is the first packet, initialize the first_timestamp_.
|
||||
if (base_timestamp_ == Timestamp::Unset()) {
|
||||
// Initialize first_timestamp_ with exactly the first packet timestamp.
|
||||
calculator_->first_timestamp_ = cc->InputTimestamp();
|
||||
} else {
|
||||
// Initialize first_timestamp_ with the first packet timestamp
|
||||
// aligned to the base_timestamp_.
|
||||
int64 first_index = MathUtil::SafeRound<int64, double>(
|
||||
(cc->InputTimestamp() - base_timestamp_).Seconds() *
|
||||
calculator_->frame_rate_);
|
||||
calculator_->first_timestamp_ =
|
||||
base_timestamp_ +
|
||||
TimestampDiffFromSeconds(first_index / calculator_->frame_rate_);
|
||||
}
|
||||
if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
|
||||
cc->Outputs()
|
||||
.Tag("VIDEO_HEADER")
|
||||
.Add(new VideoHeader(calculator_->video_header_),
|
||||
Timestamp::PreStream());
|
||||
}
|
||||
}
|
||||
const Timestamp received_timestamp = cc->InputTimestamp();
|
||||
const int64 received_timestamp_idx =
|
||||
calculator_->TimestampToPeriodIndex(received_timestamp);
|
||||
// Only consider the received packet if it belongs to the current period
|
||||
// (== period_count_) or to a newer one (> period_count_).
|
||||
if (received_timestamp_idx >= period_count_) {
|
||||
// Fill the empty periods until we are in the same index as the received
|
||||
// packet.
|
||||
while (received_timestamp_idx > period_count_) {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, calculator_->last_packet_.At(
|
||||
calculator_->PeriodIndexToTimestamp(period_count_)));
|
||||
++period_count_;
|
||||
}
|
||||
// Now, if the received packet has a timestamp larger than the middle of
|
||||
// the current period, we can send a packet without waiting. We send the
|
||||
// one closer to the middle.
|
||||
Timestamp target_timestamp =
|
||||
calculator_->PeriodIndexToTimestamp(period_count_);
|
||||
if (received_timestamp >= target_timestamp) {
|
||||
bool have_last_packet =
|
||||
(calculator_->last_packet_.Timestamp() != Timestamp::Unset());
|
||||
bool send_current =
|
||||
!have_last_packet ||
|
||||
(received_timestamp - target_timestamp <=
|
||||
target_timestamp - calculator_->last_packet_.Timestamp());
|
||||
if (send_current) {
|
||||
calculator_->OutputWithinLimits(cc,
|
||||
cc->Inputs()
|
||||
.Get(calculator_->input_data_id_)
|
||||
.Value()
|
||||
.At(target_timestamp));
|
||||
} else {
|
||||
calculator_->OutputWithinLimits(
|
||||
cc, calculator_->last_packet_.At(target_timestamp));
|
||||
}
|
||||
++period_count_;
|
||||
}
|
||||
// TODO: Add a mechanism to the framework to allow these packets
|
||||
// to be output earlier (without waiting for a much later packet to
|
||||
// arrive)
|
||||
|
||||
// Update the bound for the next packet.
|
||||
cc->Outputs()
|
||||
.Get(calculator_->output_data_id_)
|
||||
.SetNextTimestampBound(
|
||||
calculator_->PeriodIndexToTimestamp(period_count_));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -55,7 +55,7 @@ class PacketReservoir {
|
||||
// correspond to timestamp t.
|
||||
// - The next packet is chosen randomly (uniform distribution) among frames
|
||||
// that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
|
||||
// - if jitter_with_reflection_ is true, the timestamp will be reflected
|
||||
// - if jitter_with_reflection is true, the timestamp will be reflected
|
||||
// against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
|
||||
// so that its marginal distribution is uniform within this interval.
|
||||
// In the formula, t_0 is the timestamp of the first sampled
|
||||
@@ -66,6 +66,17 @@ class PacketReservoir {
|
||||
// the resampling. For Cloud ML Video Intelligence API, the hash of the
|
||||
// input video should serve this purpose. For YouTube, either video ID or
|
||||
// content hex ID of the input video should do.
|
||||
// - If reproducible_samping is true, care is taken to allow reproducible
|
||||
// "mid-stream" sampling. The calculator can be executed on a stream that
|
||||
// doesn't start at the first period. For instance, if the calculator
|
||||
// is run on a 10 second stream it will produce the same set of samples
|
||||
// as two runs of the calculator, the first with 3 seconds of input starting
|
||||
// at time 0 and the second with 7 seconds of input starting at time +3s.
|
||||
// - In order to guarantee the exact same samples, 1) the inputs must be
|
||||
// aligned with the sampling period. For instance, if the sampling rate
|
||||
// is 2 frames per second, streams should be aligned on 0.5 second
|
||||
// boundaries, and 2) the stream must include at least one extra packet
|
||||
// before and after the second aligned sampling period.
|
||||
//
|
||||
// If jitter_ is not specified:
|
||||
// - The first packet defines the first_timestamp of the output stream,
|
||||
@@ -105,19 +116,6 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Calculates the first sampled timestamp that incorporates a jittering
|
||||
// offset.
|
||||
void InitializeNextOutputTimestampWithJitter();
|
||||
// Calculates the next sampled timestamp that incorporates a jittering offset.
|
||||
void UpdateNextOutputTimestampWithJitter();
|
||||
|
||||
// Logic for Process() when jitter_ != 0.0.
|
||||
absl::Status ProcessWithJitter(CalculatorContext* cc);
|
||||
|
||||
// Logic for Process() when jitter_ == 0.0.
|
||||
absl::Status ProcessWithoutJitter(CalculatorContext* cc);
|
||||
|
||||
// Given the current count of periods that have passed, this returns
|
||||
// the next valid timestamp of the middle point of the next period:
|
||||
// if count is 0, it returns the first_timestamp_.
|
||||
@@ -141,6 +139,16 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
// Outputs a packet if it is in range (start_time_, end_time_).
|
||||
void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
|
||||
|
||||
protected:
|
||||
// Returns Sampling Strategy to use.
|
||||
//
|
||||
// Virtual to allow injection of testing strategies.
|
||||
virtual std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
|
||||
const mediapipe::PacketResamplerCalculatorOptions& options);
|
||||
|
||||
private:
|
||||
std::unique_ptr<class PacketResamplerStrategy> strategy_;
|
||||
|
||||
// The timestamp of the first packet received.
|
||||
Timestamp first_timestamp_;
|
||||
|
||||
@@ -150,14 +158,6 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
// Inverse of frame_rate_.
|
||||
int64 frame_time_usec_;
|
||||
|
||||
// Number of periods that have passed (= #packets sent to the output).
|
||||
//
|
||||
// Can only be used if jitter_ equals zero.
|
||||
int64 period_count_;
|
||||
|
||||
// The last packet that was received.
|
||||
Packet last_packet_;
|
||||
|
||||
VideoHeader video_header_;
|
||||
// The "DATA" input stream.
|
||||
CollectionItemId input_data_id_;
|
||||
@@ -165,23 +165,15 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
CollectionItemId output_data_id_;
|
||||
|
||||
// Indicator whether to flush last packet even if its timestamp is greater
|
||||
// than the final stream timestamp. Set to false when jitter_ is non-zero.
|
||||
// than the final stream timestamp.
|
||||
bool flush_last_packet_;
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
double jitter_ = 0.0;
|
||||
bool jitter_with_reflection_;
|
||||
int64 jitter_usec_;
|
||||
Timestamp next_output_timestamp_;
|
||||
// If jittering_with_reflection_ is true, next_output_timestamp_ will be
|
||||
// kept within the interval
|
||||
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
|
||||
Timestamp next_output_timestamp_min_;
|
||||
|
||||
// If specified, output timestamps are aligned with base_timestamp.
|
||||
// Otherwise, they are aligned with the first input timestamp.
|
||||
Timestamp base_timestamp_;
|
||||
int64 jitter_usec_;
|
||||
|
||||
// The last packet that was received.
|
||||
Packet last_packet_;
|
||||
|
||||
// If specified, only outputs at/after start_time are included.
|
||||
Timestamp start_time_;
|
||||
@@ -191,15 +183,210 @@ class PacketResamplerCalculator : public CalculatorBase {
|
||||
|
||||
// If set, the output timestamps nearest to start_time and end_time
|
||||
// are included in the output, even if the nearest timestamp is not
|
||||
// between start_time and end_time.W
|
||||
// between start_time and end_time.
|
||||
bool round_limits_;
|
||||
|
||||
// Allow strategies access to all internal calculator state.
|
||||
//
|
||||
// The calculator and strategies are intimiately tied together so this should
|
||||
// not break encapsulation.
|
||||
friend class LegacyJitterWithReflectionStrategy;
|
||||
friend class ReproducibleJitterWithReflectionStrategy;
|
||||
friend class JitterWithoutReflectionStrategy;
|
||||
friend class NoJitterStrategy;
|
||||
};
|
||||
|
||||
// Abstract class encapsulating sampling stategy.
|
||||
//
|
||||
// These are used solely by PacketResamplerCalculator, but are exposed here
|
||||
// to facilitate tests.
|
||||
class PacketResamplerStrategy {
|
||||
public:
|
||||
PacketResamplerStrategy(PacketResamplerCalculator* calculator)
|
||||
: calculator_(calculator) {}
|
||||
virtual ~PacketResamplerStrategy() = default;
|
||||
|
||||
// Delegate for CalculatorBase::Open. See CalculatorBase for relevant
|
||||
// implementation considerations.
|
||||
virtual absl::Status Open(CalculatorContext* cc) = 0;
|
||||
// Delegate for CalculatorBase::Close. See CalculatorBase for relevant
|
||||
// implementation considerations.
|
||||
virtual absl::Status Close(CalculatorContext* cc) = 0;
|
||||
// Delegate for CalculatorBase::Process. See CalculatorBase for relevant
|
||||
// implementation considerations.
|
||||
virtual absl::Status Process(CalculatorContext* cc) = 0;
|
||||
|
||||
protected:
|
||||
// Calculator running strategy.
|
||||
PacketResamplerCalculator* calculator_;
|
||||
};
|
||||
|
||||
// Strategy that applies Jitter with reflection based sampling.
|
||||
//
|
||||
// Used by PacketResamplerCalculator when both Jitter and reflection are
|
||||
// enabled.
|
||||
//
|
||||
// This applies the legacy jitter with reflection which doesn't allow
|
||||
// for reproducibility of sampling when starting mid-stream. This is maintained
|
||||
// for backward compatibility.
|
||||
class LegacyJitterWithReflectionStrategy : public PacketResamplerStrategy {
|
||||
public:
|
||||
LegacyJitterWithReflectionStrategy(PacketResamplerCalculator* calculator)
|
||||
: PacketResamplerStrategy(calculator) {}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
void InitializeNextOutputTimestampWithJitter();
|
||||
void UpdateNextOutputTimestampWithJitter();
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
|
||||
// The timestamp of the first packet received.
|
||||
Timestamp first_timestamp_;
|
||||
|
||||
// Next packet to be emitted. Since packets may not align perfectly with
|
||||
// next_output_timestamp_, the closest packet will be emitted.
|
||||
Timestamp next_output_timestamp_;
|
||||
|
||||
// Lower bound for next timestamp.
|
||||
//
|
||||
// next_output_timestamp_ will be kept within the interval
|
||||
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
|
||||
Timestamp next_output_timestamp_min_ = Timestamp::Unset();
|
||||
|
||||
// packet reservior used for sampling random packet out of partial
|
||||
// period when jitter is enabled
|
||||
std::unique_ptr<PacketReservoir> packet_reservoir_;
|
||||
|
||||
// random number generator used in packet_reservior_.
|
||||
std::unique_ptr<RandomBase> packet_reservoir_random_;
|
||||
};
|
||||
|
||||
// Strategy that applies reproducible jitter with reflection based sampling.
|
||||
//
|
||||
// Used by PacketResamplerCalculator when both Jitter and reflection are
|
||||
// enabled.
|
||||
class ReproducibleJitterWithReflectionStrategy
|
||||
: public PacketResamplerStrategy {
|
||||
public:
|
||||
ReproducibleJitterWithReflectionStrategy(
|
||||
PacketResamplerCalculator* calculator)
|
||||
: PacketResamplerStrategy(calculator) {}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
protected:
|
||||
// Returns next random in range (0,n].
|
||||
//
|
||||
// Exposed as virtual function for testing Jitter with reflection.
|
||||
// This is the only way random_ is accessed.
|
||||
virtual uint64 GetNextRandom(uint64 n) {
|
||||
return random_->UnbiasedUniform64(n);
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes Jitter with reflection.
|
||||
//
|
||||
// This will fast-forward to the period containing current_timestamp.
|
||||
// next_output_timestamp_ is guarnateed to be current_timestamp's period
|
||||
// and packet_emitted_this_period_ will be set to false.
|
||||
void InitializeNextOutputTimestamp(Timestamp current_timestamp);
|
||||
|
||||
// Potentially advances next_output_timestamp_ a single period.
|
||||
//
|
||||
// next_output_timestamp_ will only be advanced if packet_emitted_this_period_
|
||||
// is false. next_output_timestamp_ will never be advanced beyond
|
||||
// current_timestamp's period.
|
||||
//
|
||||
// However, next_output_timestamp_ could fall before current_timestamp's
|
||||
// period since only a single period can be advanced at a time.
|
||||
void UpdateNextOutputTimestamp(Timestamp current_timestamp);
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
|
||||
// Next packet to be emitted. Since packets may not align perfectly with
|
||||
// next_output_timestamp_, the closest packet will be emitted.
|
||||
Timestamp next_output_timestamp_;
|
||||
|
||||
// Lower bound for next timestamp.
|
||||
//
|
||||
// next_output_timestamp_ will be kept within the interval
|
||||
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
|
||||
Timestamp next_output_timestamp_min_ = Timestamp::Unset();
|
||||
|
||||
// Indicates packet was emitted for current period (i.e. the period
|
||||
// next_output_timestamp_ falls in.
|
||||
bool packet_emitted_this_period_ = false;
|
||||
};
|
||||
|
||||
// Strategy that applies Jitter without reflection based sampling.
|
||||
//
|
||||
// Used by PacketResamplerCalculator when Jitter is enabled and reflection is
|
||||
// not enabled.
|
||||
class JitterWithoutReflectionStrategy : public PacketResamplerStrategy {
|
||||
public:
|
||||
JitterWithoutReflectionStrategy(PacketResamplerCalculator* calculator)
|
||||
: PacketResamplerStrategy(calculator) {}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Calculates the first sampled timestamp that incorporates a jittering
|
||||
// offset.
|
||||
void InitializeNextOutputTimestamp();
|
||||
|
||||
// Calculates the next sampled timestamp that incorporates a jittering offset.
|
||||
void UpdateNextOutputTimestamp();
|
||||
|
||||
// Jitter-related variables.
|
||||
std::unique_ptr<RandomBase> random_;
|
||||
|
||||
// Next packet to be emitted. Since packets may not align perfectly with
|
||||
// next_output_timestamp_, the closest packet will be emitted.
|
||||
Timestamp next_output_timestamp_;
|
||||
|
||||
// Lower bound for next timestamp.
|
||||
//
|
||||
// next_output_timestamp_ will be kept within the interval
|
||||
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
|
||||
Timestamp next_output_timestamp_min_ = Timestamp::Unset();
|
||||
|
||||
// packet reservior used for sampling random packet out of partial period.
|
||||
std::unique_ptr<PacketReservoir> packet_reservoir_;
|
||||
|
||||
// random number generator used in packet_reservior_.
|
||||
std::unique_ptr<RandomBase> packet_reservoir_random_;
|
||||
};
|
||||
|
||||
// Strategy that applies sampling without any jitter.
|
||||
//
|
||||
// Used by PacketResamplerCalculator when jitter is not enabled.
|
||||
class NoJitterStrategy : public PacketResamplerStrategy {
|
||||
public:
|
||||
NoJitterStrategy(PacketResamplerCalculator* calculator)
|
||||
: PacketResamplerStrategy(calculator) {}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Number of periods that have passed (= #packets sent to the output).
|
||||
int64 period_count_;
|
||||
|
||||
// If specified, output timestamps are aligned with base_timestamp.
|
||||
// Otherwise, they are aligned with the first input timestamp.
|
||||
Timestamp base_timestamp_;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
#endif // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
|
||||
|
||||
@@ -68,8 +68,23 @@ message PacketResamplerCalculatorOptions {
|
||||
// pseudo-random number generator does its job and the number of frames is
|
||||
// sufficiently large, the average frame rate will be close to this value.
|
||||
optional double jitter = 4;
|
||||
|
||||
// Enables reflection when applying jitter.
|
||||
//
|
||||
// This option is ignored when reproducible_sampling is true, in which case
|
||||
// reflection will be used.
|
||||
//
|
||||
// New use cases should use reproducible_sampling = true, as
|
||||
// jitter_with_reflection is deprecated and will be removed at some point.
|
||||
optional bool jitter_with_reflection = 9 [default = false];
|
||||
|
||||
// If set, enabled reproducible sampling, allowing frames to be sampled
|
||||
// without regards to where the stream starts. See
|
||||
// packet_resampler_calculator.h for details.
|
||||
//
|
||||
// This enables reflection (ignoring jitter_with_reflection setting).
|
||||
optional bool reproducible_sampling = 10 [default = false];
|
||||
|
||||
// If specified, output timestamps are aligned with base_timestamp.
|
||||
// Otherwise, they are aligned with the first input timestamp.
|
||||
//
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
using ::testing::ElementsAre;
|
||||
namespace {
|
||||
// A simple version of CalculatorRunner with built-in convenience
|
||||
// methods for setting inputs from a vector and checking outputs
|
||||
@@ -96,6 +97,77 @@ class SimpleRunner : public CalculatorRunner {
|
||||
static int static_count_;
|
||||
};
|
||||
|
||||
// Matcher for Packets with uint64 payload, comparing arg packet's
|
||||
// timestamp and uint64 payload.
|
||||
MATCHER_P2(PacketAtTimestamp, payload, timestamp,
|
||||
absl::StrCat(negation ? "isn't" : "is", " a packet with payload ",
|
||||
payload, " @ time ", timestamp)) {
|
||||
if (timestamp != arg.Timestamp().Value()) {
|
||||
*result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
|
||||
return false;
|
||||
}
|
||||
int64 actual_payload = arg.template Get<int64>();
|
||||
if (actual_payload != payload) {
|
||||
*result_listener << "with incorrect payload = " << actual_payload;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// JitterWithReflectionStrategy child class which injects a specified stream
|
||||
// of "random" numbers.
|
||||
//
|
||||
// Calculators are created through factory methods, making testing and injection
|
||||
// tricky. This class utilizes a static variable, random_sequence, to pass
|
||||
// the desired random sequence into the calculator.
|
||||
class ReproducibleJitterWithReflectionStrategyForTesting
|
||||
: public ReproducibleJitterWithReflectionStrategy {
|
||||
public:
|
||||
ReproducibleJitterWithReflectionStrategyForTesting(
|
||||
PacketResamplerCalculator* calculator)
|
||||
: ReproducibleJitterWithReflectionStrategy(calculator) {}
|
||||
|
||||
// Statically accessed random sequence to use for jitter with reflection.
|
||||
//
|
||||
// An EXPECT will fail if sequence is less than the number requested during
|
||||
// processing.
|
||||
static std::vector<uint64> random_sequence;
|
||||
|
||||
protected:
|
||||
virtual uint64 GetNextRandom(uint64 n) {
|
||||
EXPECT_LT(sequence_index_, random_sequence.size());
|
||||
return random_sequence[sequence_index_++] % n;
|
||||
}
|
||||
|
||||
private:
|
||||
int32 sequence_index_ = 0;
|
||||
};
|
||||
std::vector<uint64>
|
||||
ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
|
||||
|
||||
// PacketResamplerCalculator child class which injects a specified stream
|
||||
// of "random" numbers.
|
||||
//
|
||||
// Calculators are created through factory methods, making testing and injection
|
||||
// tricky. This class utilizes a static variable, random_sequence, to pass
|
||||
// the desired random sequence into the calculator.
|
||||
class ReproducibleResamplerCalculatorForTesting
|
||||
: public PacketResamplerCalculator {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
return PacketResamplerCalculator::GetContract(cc);
|
||||
}
|
||||
|
||||
protected:
|
||||
std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
|
||||
const mediapipe::PacketResamplerCalculatorOptions& Options) {
|
||||
return absl::make_unique<
|
||||
ReproducibleJitterWithReflectionStrategyForTesting>(this);
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(ReproducibleResamplerCalculatorForTesting);
|
||||
|
||||
int SimpleRunner::static_count_ = 0;
|
||||
|
||||
TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {
|
||||
@@ -380,7 +452,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
|
||||
}
|
||||
|
||||
TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "PacketResamplerCalculator"
|
||||
input_stream: "DATA:in_data"
|
||||
input_stream: "VIDEO_HEADER:in_video_header"
|
||||
@@ -389,7 +461,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
||||
options {
|
||||
[mediapipe.PacketResamplerCalculatorOptions.ext] { frame_rate: 50.0 }
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
|
||||
runner.MutableInputs()->Tag("DATA").packets.push_back(
|
||||
@@ -633,7 +705,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
|
||||
|
||||
TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "PacketResamplerCalculator"
|
||||
input_side_packet: "OPTIONS:options"
|
||||
input_stream: "input"
|
||||
@@ -643,16 +715,16 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
frame_rate: 60
|
||||
base_timestamp: 0
|
||||
}
|
||||
})");
|
||||
})pb");
|
||||
|
||||
{
|
||||
SimpleRunner runner(node_config);
|
||||
auto options =
|
||||
new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(
|
||||
R"(
|
||||
R"pb(
|
||||
[mediapipe.PacketResamplerCalculatorOptions.ext] {
|
||||
frame_rate: 30
|
||||
})"));
|
||||
})pb"));
|
||||
runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -662,12 +734,12 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
|
||||
SimpleRunner runner(node_config);
|
||||
|
||||
auto options =
|
||||
new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"(
|
||||
new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"pb(
|
||||
merge_fields: false
|
||||
[mediapipe.PacketResamplerCalculatorOptions.ext] {
|
||||
frame_rate: 30
|
||||
base_timestamp: 0
|
||||
})"));
|
||||
})pb"));
|
||||
runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
|
||||
|
||||
runner.SetInput({-222, 15000, 32000, 49999, 150000});
|
||||
|
||||
@@ -69,7 +69,7 @@ MATCHER_P2(PairPacket, timestamp, pair, "") {
|
||||
TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
std::vector<Packet> in_prev;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -93,7 +93,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
|
||||
input_stream: 'previous2'
|
||||
output_stream: 'pair'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("pair", &graph_config_, &in_prev);
|
||||
|
||||
CalculatorGraph graph_;
|
||||
@@ -169,7 +169,7 @@ REGISTER_CALCULATOR(PacketOnCloseCalculator);
|
||||
TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
|
||||
std::vector<Packet> outputs;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -192,7 +192,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
|
||||
input_stream: 'out'
|
||||
output_stream: 'close_out'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("close_out", &graph_config_, &outputs);
|
||||
|
||||
CalculatorGraph graph_;
|
||||
@@ -231,7 +231,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
|
||||
TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
|
||||
std::vector<Packet> out_and_previous_packets;
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -253,7 +253,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
|
||||
input_stream: 'previous'
|
||||
output_stream: 'out_and_previous'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("out_and_previous", &graph_config,
|
||||
&out_and_previous_packets);
|
||||
|
||||
@@ -278,7 +278,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
|
||||
TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
|
||||
std::vector<Packet> out_and_previous_packets;
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -300,7 +300,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
|
||||
input_stream: 'previous'
|
||||
output_stream: 'out_and_previous'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("out_and_previous", &graph_config,
|
||||
&out_and_previous_packets);
|
||||
|
||||
@@ -331,7 +331,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
|
||||
TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
|
||||
std::vector<Packet> outputs;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
node {
|
||||
calculator: 'PreviousLoopbackCalculator'
|
||||
@@ -354,7 +354,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
|
||||
input_stream: 'out'
|
||||
output_stream: 'close_out'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("close_out", &graph_config_, &outputs);
|
||||
|
||||
CalculatorGraph graph_;
|
||||
@@ -386,7 +386,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
|
||||
protected:
|
||||
void SetUp() override {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'input'
|
||||
input_stream: 'force_main_empty'
|
||||
input_stream: 'force_loop_empty'
|
||||
@@ -424,7 +424,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
|
||||
input_stream: 'passed_through_prev_loop'
|
||||
output_stream: 'passed_through_input_and_prev_loop'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
|
||||
&output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config, {}));
|
||||
@@ -724,7 +724,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'input'
|
||||
# Drops "loop" when set to "true", delaying output of prev_loop, hence
|
||||
# delaying output of the graph.
|
||||
@@ -755,7 +755,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
|
||||
input_stream: 'passed_through_prev_loop'
|
||||
output_stream: 'passed_through_input_and_prev_loop'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
|
||||
&output_packets_);
|
||||
MP_ASSERT_OK(graph_.Initialize(graph_config, {}));
|
||||
|
||||
@@ -27,7 +27,7 @@ namespace mediapipe {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
@@ -53,7 +53,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -63,7 +63,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
@@ -80,7 +80,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -90,7 +90,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
min_quantized_value: 1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
@@ -107,7 +107,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -117,7 +117,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
|
||||
min_quantized_value: -1
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> empty_vector;
|
||||
runner.MutableInputs()
|
||||
@@ -133,7 +133,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -143,7 +143,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
|
||||
min_quantized_value: -64
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f};
|
||||
runner.MutableInputs()
|
||||
@@ -171,7 +171,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
|
||||
|
||||
TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "QuantizeFloatVectorCalculator"
|
||||
input_stream: "FLOAT_VECTOR:float_vector"
|
||||
output_stream: "ENCODED:encoded"
|
||||
@@ -181,7 +181,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
|
||||
min_quantized_value: -64
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
std::vector<float> vector = {-65.0f, 65.0f};
|
||||
runner.MutableInputs()
|
||||
|
||||
@@ -70,13 +70,13 @@ std::vector<T> PacketValues(const std::vector<Packet>& packets) {
|
||||
constexpr int kNumImageFrames = 5;
|
||||
constexpr int kNumFinished = 3;
|
||||
CalculatorGraphConfig::Node GetDefaultNode() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "RealTimeFlowLimiterCalculator"
|
||||
input_stream: "raw_frames"
|
||||
input_stream: "FINISHED:finished"
|
||||
input_stream_info: { tag_index: "FINISHED" back_edge: true }
|
||||
output_stream: "gated_frames"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
// Simple test to make sure that the RealTimeFlowLimiterCalculator outputs just
|
||||
@@ -219,7 +219,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
|
||||
// Back-edge "finished" limits processing to one frame in-flight.
|
||||
// The two LambdaCalculators are used to keep certain packet sets in flight.
|
||||
CalculatorGraphConfig InflightGraphConfig() {
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_1'
|
||||
input_stream: 'in_2'
|
||||
node {
|
||||
@@ -256,7 +256,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
|
||||
output_stream: 'out_1'
|
||||
output_stream: 'out_2'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
protected:
|
||||
@@ -344,7 +344,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
std::vector<Packet> a_passed;
|
||||
std::vector<Packet> b_passed;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in_a'
|
||||
input_stream: 'in_b'
|
||||
input_stream: 'finished'
|
||||
@@ -360,7 +360,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
output_stream: 'in_b_sampled'
|
||||
output_stream: 'ALLOW:allow'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::string allow_cb_name;
|
||||
tool::AddVectorSink("in_a_sampled", &graph_config_, &a_passed);
|
||||
tool::AddVectorSink("in_b_sampled", &graph_config_, &b_passed);
|
||||
@@ -442,7 +442,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
|
||||
TEST(RealTimeFlowLimiterCalculator, CanConsume) {
|
||||
std::vector<Packet> in_sampled_packets_;
|
||||
CalculatorGraphConfig graph_config_ =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: 'in'
|
||||
input_stream: 'finished'
|
||||
node {
|
||||
@@ -455,7 +455,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
|
||||
output_stream: 'in_sampled'
|
||||
output_stream: 'ALLOW:allow'
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::string allow_cb_name;
|
||||
tool::AddVectorSink("in_sampled", &graph_config_, &in_sampled_packets_);
|
||||
tool::AddCallbackCalculator("allow", &graph_config_, &allow_cb_name, true);
|
||||
|
||||
@@ -36,7 +36,7 @@ using testing::HasSubstr;
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -45,7 +45,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TICK:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -58,7 +58,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "timestamp"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -67,7 +67,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TIMESTAMP:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -79,7 +79,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -88,7 +88,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "DOES_NOT_EXIST:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -102,7 +102,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
@@ -113,7 +113,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||
output_stream: "AT_TICK:packet0"
|
||||
output_stream: "AT_PRE_STREAM:packet1"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -127,7 +127,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
node {
|
||||
@@ -136,7 +136,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||
output_stream: "AT_PRESTREAM:0:packet0"
|
||||
output_stream: "AT_PRESTREAM:1:packet1"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -149,7 +149,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
node {
|
||||
@@ -158,7 +158,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "AT_PRESTREAM:packet0"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
@@ -209,7 +209,7 @@ TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
|
||||
TEST(SidePacketToStreamCalculator, AtTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -219,7 +219,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TICK:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("packet", &graph_config, &output_packets);
|
||||
CalculatorGraph graph;
|
||||
@@ -251,7 +251,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
|
||||
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
@@ -265,7 +265,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||
output_stream: "AT_TICK:0:packet0"
|
||||
output_stream: "AT_TICK:1:packet1"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets0;
|
||||
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
|
||||
std::vector<Packet> output_packets1;
|
||||
@@ -305,7 +305,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||
TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_side_packet: "timestamp"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
@@ -315,7 +315,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TIMESTAMP:packet"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("packet", &graph_config, &output_packets);
|
||||
CalculatorGraph graph;
|
||||
@@ -337,7 +337,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||
TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_side_packet: "timestamp"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
@@ -350,7 +350,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
|
||||
output_stream: "AT_TIMESTAMP:0:packet0"
|
||||
output_stream: "AT_TIMESTAMP:1:packet1"
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets0;
|
||||
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
|
||||
std::vector<Packet> output_packets1;
|
||||
|
||||
@@ -122,7 +122,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -138,7 +138,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
@@ -171,7 +171,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -183,7 +183,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -196,7 +196,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -209,7 +209,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -223,7 +223,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -238,7 +238,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -252,7 +252,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -266,7 +266,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -281,7 +281,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -298,7 +298,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
@@ -334,7 +334,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -349,7 +349,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
|
||||
@@ -377,7 +377,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "landmarks_in"
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
@@ -394,7 +394,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
|
||||
@@ -163,7 +163,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -179,7 +179,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
@@ -214,7 +214,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -226,7 +226,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -240,7 +240,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -253,7 +253,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -269,7 +269,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -284,7 +284,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -299,7 +299,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -313,7 +313,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -330,7 +330,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -347,7 +347,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
std::vector<Packet> range_1_packets;
|
||||
@@ -385,7 +385,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -400,7 +400,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
|
||||
@@ -428,7 +428,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
@@ -445,7 +445,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -511,7 +511,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
@@ -524,7 +524,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
// Run the graph.
|
||||
CalculatorGraph graph;
|
||||
@@ -536,7 +536,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
@@ -552,7 +552,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
@@ -592,7 +592,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
@@ -609,7 +609,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
@@ -646,7 +646,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
R"pb(
|
||||
input_stream: "input_vector"
|
||||
node {
|
||||
calculator: "MovableSplitUniqueIntPtrCalculator"
|
||||
@@ -661,7 +661,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
std::vector<Packet> range_0_packets;
|
||||
tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
|
||||
|
||||
@@ -410,7 +410,9 @@ cc_library(
|
||||
srcs = ["image_properties_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
|
||||
@@ -285,7 +285,7 @@ absl::Status ImageCroppingCalculator::RenderGpu(CalculatorContext* cc) {
|
||||
|
||||
// Run cropping shader on GPU.
|
||||
{
|
||||
gpu_helper_.BindFramebuffer(dst_tex); // GL_TEXTURE0
|
||||
gpu_helper_.BindFramebuffer(dst_tex);
|
||||
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(src_tex.target(), src_tex.name());
|
||||
|
||||
@@ -41,7 +41,7 @@ constexpr char kWidthTag[] = "WIDTH";
|
||||
TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
|
||||
auto calculator_node =
|
||||
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
|
||||
R"(
|
||||
R"pb(
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_frames"
|
||||
output_stream: "IMAGE_GPU:cropped_output_frames"
|
||||
@@ -54,7 +54,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
|
||||
rotation: 0.3
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
@@ -79,7 +79,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
|
||||
TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
|
||||
auto calculator_node =
|
||||
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
|
||||
R"(
|
||||
R"pb(
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_frames"
|
||||
output_stream: "IMAGE_GPU:cropped_output_frames"
|
||||
@@ -94,7 +94,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
|
||||
rotation: 0.3
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
@@ -119,7 +119,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
|
||||
TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
|
||||
auto calculator_node =
|
||||
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
|
||||
R"(
|
||||
R"pb(
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_frames"
|
||||
input_stream: "WIDTH:crop_width"
|
||||
@@ -136,7 +136,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
|
||||
rotation: 0.3
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
@@ -168,7 +168,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
|
||||
TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
|
||||
auto calculator_node =
|
||||
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
|
||||
R"(
|
||||
R"pb(
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_frames"
|
||||
input_stream: "RECT:rect"
|
||||
@@ -184,7 +184,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
|
||||
rotation: 0.3
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
@@ -196,9 +196,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
|
||||
calculator_state.get(), inputTags, tool::CreateTagMap({}).value());
|
||||
auto& inputs = cc->Inputs();
|
||||
mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
|
||||
R"(
|
||||
R"pb(
|
||||
width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
|
||||
)");
|
||||
)pb");
|
||||
inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect);
|
||||
RectSpec expectRect = {
|
||||
.width = 1,
|
||||
|
||||
@@ -50,11 +50,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInStreams) {
|
||||
MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ImageFilePropertiesCalculator"
|
||||
input_stream: "image_bytes"
|
||||
output_stream: "properties"
|
||||
)");
|
||||
)pb");
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
@@ -79,11 +79,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInSidePackets) {
|
||||
MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ImageFilePropertiesCalculator"
|
||||
input_side_packet: "image_bytes"
|
||||
output_side_packet: "properties"
|
||||
)");
|
||||
)pb");
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableSidePackets()->Index(0) =
|
||||
@@ -108,11 +108,11 @@ TEST(ImageFilePropertiesCalculatorTest,
|
||||
MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ImageFilePropertiesCalculator"
|
||||
input_stream: "image_bytes"
|
||||
output_side_packet: "properties"
|
||||
)");
|
||||
)pb");
|
||||
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
|
||||
@@ -12,25 +12,32 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
namespace {
|
||||
constexpr char kImageFrameTag[] = "IMAGE";
|
||||
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
|
||||
} // namespace
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
#if MEDIAPIPE_DISABLE_GPU
|
||||
// Just a placeholder to not have to depend on mediapipe::GpuBuffer.
|
||||
using GpuBuffer = AnyType;
|
||||
#else
|
||||
using GpuBuffer = mediapipe::GpuBuffer;
|
||||
#endif // MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
// Extracts image properties from the input image and outputs the properties.
|
||||
// Currently only supports image size.
|
||||
// Input:
|
||||
// One of the following:
|
||||
// IMAGE: An ImageFrame
|
||||
// IMAGE: An Image or ImageFrame (for backward compatibility with existing
|
||||
// graphs that use IMAGE for ImageFrame input)
|
||||
// IMAGE_CPU: An ImageFrame
|
||||
// IMAGE_GPU: A GpuBuffer
|
||||
//
|
||||
// Output:
|
||||
@@ -42,59 +49,64 @@ namespace mediapipe {
|
||||
// input_stream: "IMAGE:image"
|
||||
// output_stream: "SIZE:size"
|
||||
// }
|
||||
class ImagePropertiesCalculator : public CalculatorBase {
|
||||
class ImagePropertiesCalculator : public Node {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
|
||||
cc->Inputs().HasTag(kGpuBufferTag));
|
||||
if (cc->Inputs().HasTag(kImageFrameTag)) {
|
||||
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
|
||||
}
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (cc->Inputs().HasTag(kGpuBufferTag)) {
|
||||
cc->Inputs().Tag(kGpuBufferTag).Set<::mediapipe::GpuBuffer>();
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
static constexpr Input<
|
||||
OneOf<mediapipe::Image, mediapipe::ImageFrame>>::Optional kIn{"IMAGE"};
|
||||
// IMAGE_CPU, dedicated to ImageFrame input, is only needed in some top-level
|
||||
// graphs for the Python Solution APIs to figure out the type of input stream
|
||||
// without running into ambiguities from IMAGE.
|
||||
// TODO: Remove IMAGE_CPU once Python Solution APIs adopt Image.
|
||||
static constexpr Input<mediapipe::ImageFrame>::Optional kInCpu{"IMAGE_CPU"};
|
||||
static constexpr Input<GpuBuffer>::Optional kInGpu{"IMAGE_GPU"};
|
||||
static constexpr Output<std::pair<int, int>> kOut{"SIZE"};
|
||||
|
||||
if (cc->Outputs().HasTag("SIZE")) {
|
||||
cc->Outputs().Tag("SIZE").Set<std::pair<int, int>>();
|
||||
}
|
||||
MEDIAPIPE_NODE_CONTRACT(kIn, kInCpu, kInGpu, kOut);
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||
RET_CHECK_EQ(kIn(cc).IsConnected() + kInCpu(cc).IsConnected() +
|
||||
kInGpu(cc).IsConnected(),
|
||||
1)
|
||||
<< "One and only one of IMAGE, IMAGE_CPU and IMAGE_GPU input is "
|
||||
"expected.";
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
int width;
|
||||
int height;
|
||||
std::pair<int, int> size;
|
||||
|
||||
if (cc->Inputs().HasTag(kImageFrameTag) &&
|
||||
!cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
|
||||
const auto& image = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
|
||||
width = image.Width();
|
||||
height = image.Height();
|
||||
if (kIn(cc).IsConnected()) {
|
||||
kIn(cc).Visit(
|
||||
[&size](const mediapipe::Image& value) {
|
||||
size.first = value.width();
|
||||
size.second = value.height();
|
||||
},
|
||||
[&size](const mediapipe::ImageFrame& value) {
|
||||
size.first = value.Width();
|
||||
size.second = value.Height();
|
||||
});
|
||||
}
|
||||
if (kInCpu(cc).IsConnected()) {
|
||||
const auto& image = *kInCpu(cc);
|
||||
size.first = image.Width();
|
||||
size.second = image.Height();
|
||||
}
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (cc->Inputs().HasTag(kGpuBufferTag) &&
|
||||
!cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
|
||||
const auto& image =
|
||||
cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
|
||||
width = image.width();
|
||||
height = image.height();
|
||||
if (kInGpu(cc).IsConnected()) {
|
||||
const auto& image = *kInGpu(cc);
|
||||
size.first = image.width();
|
||||
size.second = image.height();
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
cc->Outputs().Tag("SIZE").AddPacket(
|
||||
MakePacket<std::pair<int, int>>(width, height)
|
||||
.At(cc->InputTimestamp()));
|
||||
kOut(cc).Send(size);
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(ImagePropertiesCalculator);
|
||||
|
||||
MEDIAPIPE_REGISTER_NODE(ImagePropertiesCalculator);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -546,7 +546,7 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
|
||||
auto dst = gpu_helper_.CreateDestinationTexture(output_width, output_height,
|
||||
input.format());
|
||||
|
||||
gpu_helper_.BindFramebuffer(dst); // GL_TEXTURE0
|
||||
gpu_helper_.BindFramebuffer(dst);
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(src1.target(), src1.name());
|
||||
|
||||
|
||||
@@ -36,11 +36,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestRgbJpeg) {
|
||||
Packet input_packet = MakePacket<std::string>(contents);
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "OpenCvEncodedImageToImageFrameCalculator"
|
||||
input_stream: "encoded_image"
|
||||
output_stream: "image_frame"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
input_packet.At(Timestamp(0)));
|
||||
@@ -79,11 +79,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestGrayscaleJpeg) {
|
||||
reinterpret_cast<const char*>(&encode_buffer[0]), encode_buffer.size())));
|
||||
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "OpenCvEncodedImageToImageFrameCalculator"
|
||||
input_stream: "encoded_image"
|
||||
output_stream: "image_frame"
|
||||
)");
|
||||
)pb");
|
||||
CalculatorRunner runner(node_config);
|
||||
runner.MutableInputs()->Index(0).packets.push_back(
|
||||
input_packet.At(Timestamp(0)));
|
||||
|
||||
@@ -209,6 +209,9 @@ absl::Status RecolorCalculator::Close(CalculatorContext* cc) {
|
||||
|
||||
absl::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
|
||||
if (cc->Inputs().Tag(kMaskCpuTag).IsEmpty()) {
|
||||
cc->Outputs()
|
||||
.Tag(kImageFrameTag)
|
||||
.AddPacket(cc->Inputs().Tag(kImageFrameTag).Value());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
// Get inputs and setup output.
|
||||
@@ -270,6 +273,9 @@ absl::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
|
||||
|
||||
absl::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
|
||||
if (cc->Inputs().Tag(kMaskGpuTag).IsEmpty()) {
|
||||
cc->Outputs()
|
||||
.Tag(kGpuBufferTag)
|
||||
.AddPacket(cc->Inputs().Tag(kGpuBufferTag).Value());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
@@ -287,7 +293,7 @@ absl::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
|
||||
|
||||
// Run recolor shader on GPU.
|
||||
{
|
||||
gpu_helper_.BindFramebuffer(dst_tex); // GL_TEXTURE0
|
||||
gpu_helper_.BindFramebuffer(dst_tex);
|
||||
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(img_tex.target(), img_tex.name());
|
||||
|
||||
@@ -323,7 +323,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
|
||||
const auto& alpha_mask =
|
||||
cc->Inputs().Tag(kInputAlphaTagGpu).Get<mediapipe::GpuBuffer>();
|
||||
auto alpha_texture = gpu_helper_.CreateSourceTexture(alpha_mask);
|
||||
gpu_helper_.BindFramebuffer(output_texture); // GL_TEXTURE0
|
||||
gpu_helper_.BindFramebuffer(output_texture);
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(GL_TEXTURE_2D, input_texture.name());
|
||||
glActiveTexture(GL_TEXTURE2);
|
||||
@@ -335,7 +335,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
|
||||
glBindTexture(GL_TEXTURE_2D, 0);
|
||||
alpha_texture.Release();
|
||||
} else {
|
||||
gpu_helper_.BindFramebuffer(output_texture); // GL_TEXTURE0
|
||||
gpu_helper_.BindFramebuffer(output_texture);
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(GL_TEXTURE_2D, input_texture.name());
|
||||
GlRender(cc); // use value from options
|
||||
|
||||
@@ -490,6 +490,7 @@ cc_library(
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
|
||||
@@ -526,6 +527,7 @@ mediapipe_proto_library(
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/gpu:gpu_origin_proto",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -561,13 +563,13 @@ cc_test(
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
@@ -583,6 +585,7 @@ cc_library(
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":image_to_tensor_utils",
|
||||
"//mediapipe/framework/formats:image",
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
#include "mediapipe/gpu/gpu_origin.pb.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
@@ -236,7 +237,7 @@ class ImageToTensorCalculator : public Node {
|
||||
}
|
||||
|
||||
private:
|
||||
bool DoesInputStartAtBottom() {
|
||||
bool DoesGpuInputStartAtBottom() {
|
||||
return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
|
||||
}
|
||||
|
||||
@@ -290,11 +291,11 @@ class ImageToTensorCalculator : public Node {
|
||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
ASSIGN_OR_RETURN(gpu_converter_,
|
||||
CreateImageToGlBufferTensorConverter(
|
||||
cc, DoesInputStartAtBottom(), GetBorderMode()));
|
||||
cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
|
||||
#else
|
||||
ASSIGN_OR_RETURN(gpu_converter_,
|
||||
CreateImageToGlTextureTensorConverter(
|
||||
cc, DoesInputStartAtBottom(), GetBorderMode()));
|
||||
cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
}
|
||||
|
||||
@@ -17,20 +17,7 @@ syntax = "proto2";
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message GpuOrigin {
|
||||
enum Mode {
|
||||
DEFAULT = 0;
|
||||
|
||||
// OpenGL: bottom-left origin
|
||||
// Metal : top-left origin
|
||||
CONVENTIONAL = 1;
|
||||
|
||||
// OpenGL: top-left origin
|
||||
// Metal : top-left origin
|
||||
TOP_LEFT = 2;
|
||||
}
|
||||
}
|
||||
import "mediapipe/gpu/gpu_origin.proto";
|
||||
|
||||
message ImageToTensorCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||
@@ -28,7 +29,6 @@
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
|
||||
@@ -312,7 +312,7 @@ class GlProcessor : public ImageToTensorConverter {
|
||||
return absl::OkStatus();
|
||||
}));
|
||||
|
||||
return tensor;
|
||||
return std::move(tensor);
|
||||
}
|
||||
|
||||
~GlProcessor() override {
|
||||
|
||||
@@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter {
|
||||
tflite::gpu::HW(output_dims.height, output_dims.width),
|
||||
command_buffer, buffer_view.buffer()));
|
||||
[command_buffer commit];
|
||||
return tensor;
|
||||
return std::move(tensor);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
|
||||
GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
|
||||
range_min, range_max));
|
||||
transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset);
|
||||
return tensor;
|
||||
return std::move(tensor);
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@@ -41,7 +41,6 @@ class InferenceCalculatorSelectorImpl
|
||||
(options.has_delegate() && options.delegate().has_gpu());
|
||||
if (should_use_gpu) {
|
||||
impls.emplace_back("Metal");
|
||||
impls.emplace_back("MlDrift");
|
||||
impls.emplace_back("Gl");
|
||||
}
|
||||
impls.emplace_back("Cpu");
|
||||
|
||||
@@ -118,10 +118,6 @@ struct InferenceCalculatorGl : public InferenceCalculator {
|
||||
static constexpr char kCalculatorName[] = "InferenceCalculatorGl";
|
||||
};
|
||||
|
||||
struct InferenceCalculatorMlDrift : public InferenceCalculator {
|
||||
static constexpr char kCalculatorName[] = "InferenceCalculatorMlDrift";
|
||||
};
|
||||
|
||||
struct InferenceCalculatorMetal : public InferenceCalculator {
|
||||
static constexpr char kCalculatorName[] = "InferenceCalculatorMetal";
|
||||
};
|
||||
|
||||
@@ -51,12 +51,12 @@ message InferenceCalculatorOptions {
|
||||
|
||||
// This option is valid for TFLite GPU delegate API2 only,
|
||||
// Choose any of available APIs to force running inference using it.
|
||||
enum API {
|
||||
enum Api {
|
||||
ANY = 0;
|
||||
OPENGL = 1;
|
||||
OPENCL = 2;
|
||||
}
|
||||
optional API api = 4 [default = ANY];
|
||||
optional Api api = 4 [default = ANY];
|
||||
|
||||
// This option is valid for TFLite GPU delegate API2 only,
|
||||
// Set to true to use 16-bit float precision. If max precision is needed,
|
||||
|
||||
@@ -136,7 +136,7 @@ absl::Status InferenceCalculatorCpuImpl::LoadModel(CalculatorContext* cc) {
|
||||
const auto& model = *model_packet_.Get();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver =
|
||||
kSideInCustomOpResolver(cc).GetOr(
|
||||
tflite::ops::builtin::BuiltinOpResolver());
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
|
||||
|
||||
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
|
||||
RET_CHECK(interpreter_);
|
||||
|
||||
@@ -57,11 +57,7 @@ const std::vector<Param>& GetParams() {
|
||||
// Metal is not available on the iOS simulator.
|
||||
p.push_back({"Metal", "Metal"});
|
||||
p.back().delegate.mutable_gpu();
|
||||
#endif // TARGET_IPHONE_SIMULATOR
|
||||
#if __EMSCRIPTEN__
|
||||
p.push_back({"MlDrift", "MlDrift"});
|
||||
p.back().delegate.mutable_gpu();
|
||||
#endif // __EMSCRIPTEN__
|
||||
#endif // TARGET_IPHONE_SIMULATOR
|
||||
#if __ANDROID__ && 0 // Disabled for now since emulator can't go GLESv3
|
||||
p.push_back({"Gl", "Gl"});
|
||||
p.back().delegate.mutable_gpu();
|
||||
@@ -78,18 +74,6 @@ const std::vector<Param>& GetParams() {
|
||||
|
||||
class InferenceCalculatorTest : public testing::TestWithParam<Param> {
|
||||
protected:
|
||||
#if __EMSCRIPTEN__
|
||||
// TODO: fix Tensor locking.
|
||||
// The MlDrift backend currently fails in debug mode without this,
|
||||
// because of Tensor locking issues. I am adding this temporarily since
|
||||
// the calculator is already being used and it's better to have test
|
||||
// coverage for it. Also, the issue doesn't apply to our Emscripten
|
||||
// build in practice since it's single-threaded.
|
||||
void SetUp(void) override {
|
||||
absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kIgnore);
|
||||
}
|
||||
#endif // __EMSCRIPTEN__
|
||||
|
||||
void SetDelegateForParam(mediapipe::CalculatorGraphConfig_Node* node) {
|
||||
*node->mutable_options()
|
||||
->MutableExtension(mediapipe::InferenceCalculatorOptions::ext)
|
||||
|
||||
@@ -63,7 +63,7 @@ class InferenceCalculatorGlImpl
|
||||
mediapipe::GlCalculatorHelper gpu_helper_;
|
||||
std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
|
||||
bool allow_precision_loss_ = false;
|
||||
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::API
|
||||
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
|
||||
tflite_gpu_runner_api_;
|
||||
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
|
||||
|
||||
@@ -244,7 +244,7 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
|
||||
const auto& model = *model_packet_.Get();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver =
|
||||
kSideInCustomOpResolver(cc).GetOr(
|
||||
tflite::ops::builtin::BuiltinOpResolver());
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
|
||||
|
||||
// Create runner
|
||||
tflite::gpu::InferenceOptions options;
|
||||
@@ -294,7 +294,7 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
|
||||
const auto& model = *model_packet_.Get();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver =
|
||||
kSideInCustomOpResolver(cc).GetOr(
|
||||
tflite::ops::builtin::BuiltinOpResolver());
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
|
||||
|
||||
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
|
||||
RET_CHECK(interpreter_);
|
||||
@@ -317,7 +317,8 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
|
||||
absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
|
||||
// Configure and create the delegate.
|
||||
TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault();
|
||||
options.compile_options.precision_loss_allowed = 1;
|
||||
options.compile_options.precision_loss_allowed =
|
||||
allow_precision_loss_ ? 1 : 0;
|
||||
options.compile_options.preferred_gl_object_type =
|
||||
TFLITE_GL_OBJECT_TYPE_FASTEST;
|
||||
options.compile_options.dynamic_batch_enabled = 0;
|
||||
|
||||
@@ -97,6 +97,7 @@ class InferenceCalculatorMetalImpl
|
||||
Packet<TfLiteModelPtr> model_packet_;
|
||||
std::unique_ptr<tflite::Interpreter> interpreter_;
|
||||
TfLiteDelegatePtr delegate_;
|
||||
bool allow_precision_loss_ = false;
|
||||
|
||||
#if MEDIAPIPE_TFLITE_METAL_INFERENCE
|
||||
MPPMetalHelper* gpu_helper_ = nullptr;
|
||||
@@ -122,6 +123,9 @@ absl::Status InferenceCalculatorMetalImpl::UpdateContract(
|
||||
}
|
||||
|
||||
absl::Status InferenceCalculatorMetalImpl::Open(CalculatorContext* cc) {
|
||||
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
|
||||
allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
|
||||
|
||||
MP_RETURN_IF_ERROR(LoadModel(cc));
|
||||
|
||||
gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
|
||||
@@ -200,7 +204,7 @@ absl::Status InferenceCalculatorMetalImpl::LoadModel(CalculatorContext* cc) {
|
||||
const auto& model = *model_packet_.Get();
|
||||
tflite::ops::builtin::BuiltinOpResolver op_resolver =
|
||||
kSideInCustomOpResolver(cc).GetOr(
|
||||
tflite::ops::builtin::BuiltinOpResolver());
|
||||
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
|
||||
|
||||
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
|
||||
RET_CHECK(interpreter_);
|
||||
@@ -222,7 +226,7 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
|
||||
|
||||
// Configure and create the delegate.
|
||||
TFLGpuDelegateOptions options;
|
||||
options.allow_precision_loss = true;
|
||||
options.allow_precision_loss = allow_precision_loss_;
|
||||
options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypeDoNotWait;
|
||||
delegate_ =
|
||||
TfLiteDelegatePtr(TFLGpuDelegateCreate(&options), &TFLGpuDelegateDelete);
|
||||
@@ -239,7 +243,9 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
|
||||
tensor->dims->data + tensor->dims->size};
|
||||
dims.back() = RoundUp(dims.back(), 4);
|
||||
gpu_buffers_in_.emplace_back(absl::make_unique<Tensor>(
|
||||
Tensor::ElementType::kFloat16, Tensor::Shape{dims}));
|
||||
allow_precision_loss_ ? Tensor::ElementType::kFloat16
|
||||
: Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{dims}));
|
||||
auto buffer_view =
|
||||
gpu_buffers_in_[i]->GetMtlBufferWriteView(gpu_helper_.mtlDevice);
|
||||
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
|
||||
@@ -261,7 +267,9 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
|
||||
output_shapes_[i] = {dims};
|
||||
dims.back() = RoundUp(dims.back(), 4);
|
||||
gpu_buffers_out_.emplace_back(absl::make_unique<Tensor>(
|
||||
Tensor::ElementType::kFloat16, Tensor::Shape{dims}));
|
||||
allow_precision_loss_ ? Tensor::ElementType::kFloat16
|
||||
: Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{dims}));
|
||||
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
|
||||
delegate_.get(), output_indices[i],
|
||||
gpu_buffers_out_[i]
|
||||
@@ -271,17 +279,19 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
|
||||
}
|
||||
|
||||
// Create converter for GPU input.
|
||||
converter_to_BPHWC4_ = [[TFLBufferConvert alloc] initWithDevice:device
|
||||
isFloat16:true
|
||||
convertToPBHWC4:true];
|
||||
converter_to_BPHWC4_ =
|
||||
[[TFLBufferConvert alloc] initWithDevice:device
|
||||
isFloat16:allow_precision_loss_
|
||||
convertToPBHWC4:true];
|
||||
if (converter_to_BPHWC4_ == nil) {
|
||||
return mediapipe::InternalError(
|
||||
"Error initializating input buffer converter");
|
||||
}
|
||||
// Create converter for GPU output.
|
||||
converter_from_BPHWC4_ = [[TFLBufferConvert alloc] initWithDevice:device
|
||||
isFloat16:true
|
||||
convertToPBHWC4:false];
|
||||
converter_from_BPHWC4_ =
|
||||
[[TFLBufferConvert alloc] initWithDevice:device
|
||||
isFloat16:allow_precision_loss_
|
||||
convertToPBHWC4:false];
|
||||
if (converter_from_BPHWC4_ == nil) {
|
||||
return absl::InternalError("Error initializating output buffer converter");
|
||||
}
|
||||
|
||||
@@ -84,7 +84,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
|
||||
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
@@ -96,7 +96,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
@@ -146,7 +146,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
|
||||
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "matrix"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
@@ -158,7 +158,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
@@ -205,7 +205,7 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
|
||||
CalculatorGraph graph;
|
||||
// Run the calculator and verify that one output is generated.
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
|
||||
input_stream: "input_image"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
@@ -220,7 +220,7 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
|
||||
@@ -89,7 +89,8 @@ absl::Status TensorsToClassificationCalculator::Open(CalculatorContext* cc) {
|
||||
ASSIGN_OR_RETURN(string_path,
|
||||
PathToResourceAsFile(options_.label_map_path()));
|
||||
std::string label_map_string;
|
||||
MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
|
||||
MP_RETURN_IF_ERROR(
|
||||
mediapipe::GetResourceContents(string_path, &label_map_string));
|
||||
|
||||
std::istringstream stream(label_map_string);
|
||||
std::string line;
|
||||
@@ -98,6 +99,14 @@ absl::Status TensorsToClassificationCalculator::Open(CalculatorContext* cc) {
|
||||
label_map_[i++] = line;
|
||||
}
|
||||
label_map_loaded_ = true;
|
||||
} else if (options_.has_label_map()) {
|
||||
for (int i = 0; i < options_.label_map().entries_size(); ++i) {
|
||||
const auto& entry = options_.label_map().entries(i);
|
||||
RET_CHECK(!label_map_.contains(entry.id()))
|
||||
<< "Duplicate id found: " << entry.id();
|
||||
label_map_[entry.id()] = entry.label();
|
||||
}
|
||||
label_map_loaded_ = true;
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
|
||||
@@ -25,6 +25,14 @@ message TensorsToClassificationCalculatorOptions {
|
||||
optional TensorsToClassificationCalculatorOptions ext = 335742638;
|
||||
}
|
||||
|
||||
message LabelMap {
|
||||
message Entry {
|
||||
optional int32 id = 1;
|
||||
optional string label = 2;
|
||||
}
|
||||
repeated Entry entries = 1;
|
||||
}
|
||||
|
||||
// Score threshold for perserving the class.
|
||||
optional float min_score_threshold = 1;
|
||||
// Number of highest scoring labels to output. If top_k is not positive then
|
||||
@@ -32,6 +40,10 @@ message TensorsToClassificationCalculatorOptions {
|
||||
optional int32 top_k = 2;
|
||||
// Path to a label map file for getting the actual name of class ids.
|
||||
optional string label_map_path = 3;
|
||||
// Label map. (Can be used instead of label_map_path.)
|
||||
// NOTE: "label_map_path", if specified, takes precedence over "label_map".
|
||||
optional LabelMap label_map = 5;
|
||||
|
||||
// Whether the input is a single float for binary classification.
|
||||
// When true, only a single float is expected in the input tensor and the
|
||||
// label map, if provided, is expected to have exactly two labels.
|
||||
|
||||
@@ -56,14 +56,14 @@ class TensorsToClassificationCalculatorTest : public ::testing::Test {
|
||||
};
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -85,7 +85,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) {
|
||||
}
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
@@ -94,7 +94,42 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
|
||||
label_map_path: "mediapipe/calculators/tensor/testdata/labelmap.txt"
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
|
||||
|
||||
EXPECT_EQ(1, output_packets_.size());
|
||||
|
||||
const auto& classification_list =
|
||||
output_packets_[0].Get<ClassificationList>();
|
||||
EXPECT_EQ(3, classification_list.classification_size());
|
||||
|
||||
// Verify that the label field is set.
|
||||
for (int i = 0; i < classification_list.classification_size(); ++i) {
|
||||
EXPECT_EQ(i, classification_list.classification(i).index());
|
||||
EXPECT_EQ(i * 0.5, classification_list.classification(i).score());
|
||||
ASSERT_TRUE(classification_list.classification(i).has_label());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMap) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {
|
||||
label_map {
|
||||
entries { id: 0, label: "ClassA" }
|
||||
entries { id: 1, label: "ClassB" }
|
||||
entries { id: 2, label: "ClassC" }
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -117,7 +152,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest,
|
||||
CorrectOutputWithLabelMinScoreThreshold) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
@@ -126,7 +161,7 @@ TEST_F(TensorsToClassificationCalculatorTest,
|
||||
min_score_threshold: 0.6
|
||||
}
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
@@ -144,14 +179,14 @@ TEST_F(TensorsToClassificationCalculatorTest,
|
||||
}
|
||||
|
||||
TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "CLASSIFICATIONS:classifications"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] { top_k: 2 }
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
BuildGraph(&runner, {0, 0.5, 1});
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
@@ -57,11 +57,11 @@ class TensorsToFloatsCalculatorTest : public ::testing::Test {
|
||||
};
|
||||
|
||||
TEST_F(TensorsToFloatsCalculatorTest, SingleValue) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "FLOAT:float"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
const float single_value = 0.5;
|
||||
BuildGraph(&runner, {single_value});
|
||||
@@ -76,11 +76,11 @@ TEST_F(TensorsToFloatsCalculatorTest, SingleValue) {
|
||||
}
|
||||
|
||||
TEST_F(TensorsToFloatsCalculatorTest, SingleValueAsVector) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "FLOATS:floats"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
const float single_value = 0.5;
|
||||
BuildGraph(&runner, {single_value});
|
||||
@@ -95,11 +95,11 @@ TEST_F(TensorsToFloatsCalculatorTest, SingleValueAsVector) {
|
||||
}
|
||||
|
||||
TEST_F(TensorsToFloatsCalculatorTest, FloatVector) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "FLOATS:floats"
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
const std::vector<float> input_values = {0.f, 0.5f, 1.0f};
|
||||
BuildGraph(&runner, input_values);
|
||||
@@ -116,14 +116,14 @@ TEST_F(TensorsToFloatsCalculatorTest, FloatVector) {
|
||||
}
|
||||
|
||||
TEST_F(TensorsToFloatsCalculatorTest, FloatVectorWithSigmoid) {
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
|
||||
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "FLOATS:floats"
|
||||
options {
|
||||
[mediapipe.TensorsToFloatsCalculatorOptions.ext] { activation: SIGMOID }
|
||||
}
|
||||
)"));
|
||||
)pb"));
|
||||
|
||||
const std::vector<float> input_values = {-1.f, 0.f, 1.0f};
|
||||
const std::vector<float> expected_output_with_sigmoid = {0.269f, 0.5f,
|
||||
|
||||
@@ -892,13 +892,13 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:tag_map_helper",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
@@ -923,13 +923,13 @@ cc_test(
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework:packet_generator_cc_proto",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:tag_map_helper",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
@@ -954,11 +954,11 @@ cc_test(
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework:packet_generator_cc_proto",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:tag_map_helper",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:all_kernels",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
@@ -981,11 +981,11 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:tag_map_helper",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:all_kernels",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
@@ -1144,8 +1144,8 @@ cc_test(
|
||||
":tensorflow_inference_calculator",
|
||||
":tensorflow_session_from_frozen_graph_generator",
|
||||
":tensorflow_session_from_frozen_graph_generator_cc_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"//mediapipe/framework/deps:file_path",
|
||||
"//mediapipe/framework/port:commandlineflags",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
|
||||
+8
-8
@@ -68,31 +68,31 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
|
||||
|
||||
void CreateNodeConfig(CalculatorGraphConfig::Node* node_config) const {
|
||||
ASSERT_NE(nullptr, node_config);
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ObjectDetectionTensorsToDetectionsCalculator"
|
||||
input_stream: "NUM_DETECTIONS:num_detections"
|
||||
input_stream: "BOXES:boxes"
|
||||
input_stream: "SCORES:scores"
|
||||
input_stream: "CLASSES:classes"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
void CreateNodeConfigRawTensors(
|
||||
CalculatorGraphConfig::Node* node_config) const {
|
||||
ASSERT_NE(nullptr, node_config);
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ObjectDetectionTensorsToDetectionsCalculator"
|
||||
input_stream: "BOXES:raw_detection_boxes"
|
||||
input_stream: "SCORES:raw_detection_scores"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
void CreateNodeConfigWithKeypoints(
|
||||
CalculatorGraphConfig::Node* node_config) const {
|
||||
ASSERT_NE(nullptr, node_config);
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
*node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ObjectDetectionTensorsToDetectionsCalculator"
|
||||
input_stream: "NUM_DETECTIONS:num_detections"
|
||||
input_stream: "BOXES:boxes"
|
||||
@@ -100,7 +100,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
|
||||
input_stream: "CLASSES:classes"
|
||||
input_stream: "KEYPOINTS:keypoints"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
)");
|
||||
)pb");
|
||||
}
|
||||
|
||||
void SetUpCalculatorRunner() {
|
||||
@@ -177,7 +177,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
|
||||
InsertExtraSingltonDim(&input_scores_);
|
||||
InsertExtraSingltonDim(&input_classes_);
|
||||
CalculatorGraphConfig::Node node_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
|
||||
calculator: "ObjectDetectionTensorsToDetectionsCalculator"
|
||||
input_stream: "NUM_DETECTIONS:num_detections"
|
||||
input_stream: "BOXES:boxes"
|
||||
@@ -188,7 +188,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
|
||||
[mediapipe.ObjectDetectionsTensorToDetectionsCalculatorOptions
|
||||
.ext]: { tensor_dim_to_squeeze: 0 }
|
||||
}
|
||||
)");
|
||||
)pb");
|
||||
runner_ = absl::make_unique<CalculatorRunner>(node_config);
|
||||
runner_->MutableInputs()
|
||||
->Tag(kNumDetections)
|
||||
|
||||
@@ -267,6 +267,17 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
}
|
||||
}
|
||||
|
||||
absl::Status VerifySize() {
|
||||
const int64 MAX_PROTO_BYTES = 1073741823;
|
||||
std::string id = mpms::HasExampleId(*sequence_)
|
||||
? mpms::GetExampleId(*sequence_)
|
||||
: "example";
|
||||
RET_CHECK_LT(sequence_->ByteSizeLong(), MAX_PROTO_BYTES)
|
||||
<< "sequence '" << id
|
||||
<< "' would be too many bytes to serialize after adding features.";
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Close(CalculatorContext* cc) override {
|
||||
auto& options = cc->Options<PackMediaSequenceCalculatorOptions>();
|
||||
if (options.reconcile_metadata()) {
|
||||
@@ -275,6 +286,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
options.reconcile_region_annotations(), sequence_.get()));
|
||||
}
|
||||
|
||||
if (options.skip_large_sequences()) {
|
||||
RET_CHECK_OK(VerifySize());
|
||||
}
|
||||
if (options.output_only_if_all_present()) {
|
||||
absl::Status status = VerifySequence();
|
||||
if (!status.ok()) {
|
||||
|
||||
@@ -61,4 +61,8 @@ message PackMediaSequenceCalculatorOptions {
|
||||
// present, the previous images and timestamps will be removed before adding
|
||||
// the new images.
|
||||
optional bool replace_data_instead_of_append = 4 [default = true];
|
||||
|
||||
// If true, will return an error status if an output sequence would be too
|
||||
// many bytes to serialize.
|
||||
optional bool skip_large_sequences = 7 [default = true];
|
||||
}
|
||||
|
||||
@@ -889,5 +889,24 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
|
||||
MP_ASSERT_OK(runner_->Run());
|
||||
}
|
||||
|
||||
TEST_F(PackMediaSequenceCalculatorTest, TestTooLargeInputFailsSoftly) {
|
||||
SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true);
|
||||
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
|
||||
|
||||
// 1 billion floats should be > 1GB which can't be serialized. It should fail
|
||||
// gracefully with this input.
|
||||
int num_timesteps = 1000;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto vf_ptr = ::absl::make_unique<std::vector<float>>(1000000, i);
|
||||
runner_->MutableInputs()
|
||||
->Tag("FLOAT_FEATURE_TEST")
|
||||
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
|
||||
}
|
||||
|
||||
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
|
||||
Adopt(input_sequence.release());
|
||||
ASSERT_FALSE(runner_->Run().ok());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -34,15 +34,28 @@ constexpr char kTensor[] = "TENSOR";
|
||||
} // namespace
|
||||
|
||||
// Input:
|
||||
// Tensor of type DT_FLOAT, with values between 0-255 (SRGB or GRAY8). The
|
||||
// shape can be HxWx{3,1} or simply HxW.
|
||||
// Tensor of type DT_FLOAT or DT_UINT8, with values between 0-255
|
||||
// (SRGB or GRAY8). The shape can be HxWx{3,1} or simply HxW.
|
||||
//
|
||||
// Optionally supports a scale factor that can scale 0-1 value ranges to 0-255.
|
||||
// For DT_FLOAT tensors, optionally supports a scale factor that can scale 0-1
|
||||
// value ranges to 0-255.
|
||||
//
|
||||
// Output:
|
||||
// ImageFrame containing the values of the tensor cast as uint8 (SRGB or GRAY8)
|
||||
//
|
||||
// Possible extensions: support other input ranges, maybe 4D tensors.
|
||||
//
|
||||
// Example:
|
||||
// node {
|
||||
// calculator: "TensorToImageFrameCalculator"
|
||||
// input_stream: "TENSOR:3d_float_tensor"
|
||||
// output_stream: "IMAGE:image_frame"
|
||||
// options {
|
||||
// [mediapipe.TensorToImageFrameCalculatorOptions.ext] {
|
||||
// scale_factor: 1.0 # set to 255.0 for [0,1] -> [0,255] scaling
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class TensorToImageFrameCalculator : public CalculatorBase {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
@@ -57,8 +70,8 @@ class TensorToImageFrameCalculator : public CalculatorBase {
|
||||
REGISTER_CALCULATOR(TensorToImageFrameCalculator);
|
||||
|
||||
absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) {
|
||||
RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
|
||||
<< "Only one input stream is supported.";
|
||||
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
|
||||
<< "Only one output stream is supported.";
|
||||
RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
|
||||
<< "One input stream must be provided.";
|
||||
RET_CHECK(cc->Inputs().HasTag(kTensor))
|
||||
@@ -91,29 +104,44 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) {
|
||||
RET_CHECK_EQ(depth, 3) << "Output tensor depth must be 3 or 1.";
|
||||
}
|
||||
}
|
||||
const int32 total_size =
|
||||
input_tensor.dim_size(0) * input_tensor.dim_size(1) * depth;
|
||||
std::unique_ptr<uint8[]> buffer(new uint8[total_size]);
|
||||
auto data = input_tensor.flat<float>().data();
|
||||
for (int i = 0; i < total_size; ++i) {
|
||||
float d = scale_factor_ * data[i];
|
||||
if (d < 0) d = 0;
|
||||
if (d > 255) d = 255;
|
||||
buffer[i] = d;
|
||||
int32 height = input_tensor.dim_size(0);
|
||||
int32 width = input_tensor.dim_size(1);
|
||||
auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8);
|
||||
const int32 total_size = height * width * depth;
|
||||
|
||||
::std::unique_ptr<const ImageFrame> output;
|
||||
if (input_tensor.dtype() == tensorflow::DT_FLOAT) {
|
||||
// Allocate buffer with alignments.
|
||||
std::unique_ptr<uint8_t[]> buffer(
|
||||
new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]);
|
||||
auto data = input_tensor.flat<float>().data();
|
||||
for (int i = 0; i < total_size; ++i) {
|
||||
float d = scale_factor_ * data[i];
|
||||
if (d < 0) d = 0;
|
||||
if (d > 255) d = 255;
|
||||
buffer[i] = d;
|
||||
}
|
||||
output = ::absl::make_unique<ImageFrame>(format, width, height,
|
||||
width * depth, buffer.release());
|
||||
} else if (input_tensor.dtype() == tensorflow::DT_UINT8) {
|
||||
if (scale_factor_ != 1.0) {
|
||||
return absl::InvalidArgumentError("scale_factor_ given for uint8 tensor");
|
||||
}
|
||||
// tf::Tensor has internally ref-counted buffer. The following code make the
|
||||
// ImageFrame own the copied Tensor through the deleter, which increases
|
||||
// the refcount of the buffer and allow us to use the shared buffer as the
|
||||
// image. This allows us to create an ImageFrame object without copying
|
||||
// buffer. const ImageFrame prevents the buffer from being modified later.
|
||||
auto copy = new tf::Tensor(input_tensor);
|
||||
output = ::absl::make_unique<const ImageFrame>(
|
||||
format, width, height, width * depth, copy->flat<uint8_t>().data(),
|
||||
[copy](uint8*) { delete copy; });
|
||||
} else {
|
||||
return absl::InvalidArgumentError(
|
||||
absl::StrCat("Expected float or uint8 tensor, received ",
|
||||
DataTypeString(input_tensor.dtype())));
|
||||
}
|
||||
|
||||
::std::unique_ptr<ImageFrame> output;
|
||||
if (depth == 3) {
|
||||
output = ::absl::make_unique<ImageFrame>(
|
||||
ImageFormat::SRGB, input_tensor.dim_size(1), input_tensor.dim_size(0),
|
||||
input_tensor.dim_size(1) * 3, buffer.release());
|
||||
} else if (depth == 1) {
|
||||
output = ::absl::make_unique<ImageFrame>(
|
||||
ImageFormat::GRAY8, input_tensor.dim_size(1), input_tensor.dim_size(0),
|
||||
input_tensor.dim_size(1), buffer.release());
|
||||
} else {
|
||||
return absl::InvalidArgumentError("Unrecognized image depth.");
|
||||
}
|
||||
cc->Outputs().Tag(kImage).Add(output.release(), cc->InputTimestamp());
|
||||
|
||||
return absl::OkStatus();
|
||||
|
||||
@@ -29,6 +29,7 @@ constexpr char kImage[] = "IMAGE";
|
||||
|
||||
} // namespace
|
||||
|
||||
template <class TypeParam>
|
||||
class TensorToImageFrameCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUpRunner() {
|
||||
@@ -42,14 +43,20 @@ class TensorToImageFrameCalculatorTest : public ::testing::Test {
|
||||
std::unique_ptr<CalculatorRunner> runner_;
|
||||
};
|
||||
|
||||
TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame) {
|
||||
SetUpRunner();
|
||||
using TensorToImageFrameCalculatorTestTypes = ::testing::Types<float, uint8_t>;
|
||||
TYPED_TEST_CASE(TensorToImageFrameCalculatorTest,
|
||||
TensorToImageFrameCalculatorTestTypes);
|
||||
|
||||
TYPED_TEST(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame) {
|
||||
// TYPED_TEST requires explicit "this->"
|
||||
this->SetUpRunner();
|
||||
auto& runner = this->runner_;
|
||||
constexpr int kWidth = 16;
|
||||
constexpr int kHeight = 8;
|
||||
const tf::TensorShape tensor_shape(
|
||||
std::vector<tf::int64>{kHeight, kWidth, 3});
|
||||
auto tensor = absl::make_unique<tf::Tensor>(tf::DT_FLOAT, tensor_shape);
|
||||
auto tensor_vec = tensor->flat<float>().data();
|
||||
const tf::TensorShape tensor_shape{kHeight, kWidth, 3};
|
||||
auto tensor = absl::make_unique<tf::Tensor>(
|
||||
tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
|
||||
auto tensor_vec = tensor->template flat<TypeParam>().data();
|
||||
|
||||
// Writing sequence of integers as floats which we want back (as they were
|
||||
// written).
|
||||
@@ -58,15 +65,16 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame) {
|
||||
}
|
||||
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()->Tag(kTensor).packets.push_back(
|
||||
runner->MutableInputs()->Tag(kTensor).packets.push_back(
|
||||
Adopt(tensor.release()).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
EXPECT_TRUE(runner->Run().ok());
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag(kImage).packets;
|
||||
runner->Outputs().Tag(kImage).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
|
||||
EXPECT_EQ(ImageFormat::SRGB, output_image.Format());
|
||||
EXPECT_EQ(kWidth, output_image.Width());
|
||||
EXPECT_EQ(kHeight, output_image.Height());
|
||||
|
||||
@@ -76,14 +84,15 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrameGray) {
|
||||
SetUpRunner();
|
||||
TYPED_TEST(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrameGray) {
|
||||
this->SetUpRunner();
|
||||
auto& runner = this->runner_;
|
||||
constexpr int kWidth = 16;
|
||||
constexpr int kHeight = 8;
|
||||
const tf::TensorShape tensor_shape(
|
||||
std::vector<tf::int64>{kHeight, kWidth, 1});
|
||||
auto tensor = absl::make_unique<tf::Tensor>(tf::DT_FLOAT, tensor_shape);
|
||||
auto tensor_vec = tensor->flat<float>().data();
|
||||
const tf::TensorShape tensor_shape{kHeight, kWidth, 1};
|
||||
auto tensor = absl::make_unique<tf::Tensor>(
|
||||
tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
|
||||
auto tensor_vec = tensor->template flat<TypeParam>().data();
|
||||
|
||||
// Writing sequence of integers as floats which we want back (as they were
|
||||
// written).
|
||||
@@ -92,15 +101,16 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrameGray) {
|
||||
}
|
||||
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()->Tag(kTensor).packets.push_back(
|
||||
runner->MutableInputs()->Tag(kTensor).packets.push_back(
|
||||
Adopt(tensor.release()).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
EXPECT_TRUE(runner->Run().ok());
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag(kImage).packets;
|
||||
runner->Outputs().Tag(kImage).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
|
||||
EXPECT_EQ(ImageFormat::GRAY8, output_image.Format());
|
||||
EXPECT_EQ(kWidth, output_image.Width());
|
||||
EXPECT_EQ(kHeight, output_image.Height());
|
||||
|
||||
@@ -110,13 +120,16 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrameGray) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame2DGray) {
|
||||
SetUpRunner();
|
||||
TYPED_TEST(TensorToImageFrameCalculatorTest,
|
||||
Converts3DTensorToImageFrame2DGray) {
|
||||
this->SetUpRunner();
|
||||
auto& runner = this->runner_;
|
||||
constexpr int kWidth = 16;
|
||||
constexpr int kHeight = 8;
|
||||
const tf::TensorShape tensor_shape(std::vector<tf::int64>{kHeight, kWidth});
|
||||
auto tensor = absl::make_unique<tf::Tensor>(tf::DT_FLOAT, tensor_shape);
|
||||
auto tensor_vec = tensor->flat<float>().data();
|
||||
const tf::TensorShape tensor_shape{kHeight, kWidth};
|
||||
auto tensor = absl::make_unique<tf::Tensor>(
|
||||
tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
|
||||
auto tensor_vec = tensor->template flat<TypeParam>().data();
|
||||
|
||||
// Writing sequence of integers as floats which we want back (as they were
|
||||
// written).
|
||||
@@ -125,15 +138,16 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame2DGray) {
|
||||
}
|
||||
|
||||
const int64 time = 1234;
|
||||
runner_->MutableInputs()->Tag(kTensor).packets.push_back(
|
||||
runner->MutableInputs()->Tag(kTensor).packets.push_back(
|
||||
Adopt(tensor.release()).At(Timestamp(time)));
|
||||
|
||||
EXPECT_TRUE(runner_->Run().ok());
|
||||
EXPECT_TRUE(runner->Run().ok());
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Tag(kImage).packets;
|
||||
runner->Outputs().Tag(kImage).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
|
||||
EXPECT_EQ(ImageFormat::GRAY8, output_image.Format());
|
||||
EXPECT_EQ(kWidth, output_image.Width());
|
||||
EXPECT_EQ(kHeight, output_image.Height());
|
||||
|
||||
|
||||
@@ -91,8 +91,6 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
|
||||
// the input data when it arrives in Process(). In particular, if the header
|
||||
// states that we produce a 1xD column vector, the input tensor must also be 1xD
|
||||
//
|
||||
// This designed was discussed in http://g/speakeranalysis/4uyx7cNRwJY and
|
||||
// http://g/daredevil-project/VB26tcseUy8.
|
||||
// Example Config
|
||||
// node: {
|
||||
// calculator: "TensorToMatrixCalculator"
|
||||
@@ -158,22 +156,17 @@ absl::Status TensorToMatrixCalculator::Open(CalculatorContext* cc) {
|
||||
if (header_status.ok()) {
|
||||
if (cc->Options<TensorToMatrixCalculatorOptions>()
|
||||
.has_time_series_header_overrides()) {
|
||||
// From design discussions with Daredevil, we only want to support single
|
||||
// sample per packet for now, so we hardcode the sample_rate based on the
|
||||
// packet_rate of the REFERENCE and fail noisily if we cannot. An
|
||||
// alternative would be to calculate the sample_rate from the reference
|
||||
// sample_rate and the change in num_samples between the reference and
|
||||
// override headers:
|
||||
// sample_rate_output = sample_rate_reference /
|
||||
// (num_samples_override / num_samples_reference)
|
||||
// This only supports a single sample per packet for now, so we hardcode
|
||||
// the sample_rate based on the packet_rate of the REFERENCE and fail
|
||||
// if we cannot.
|
||||
const TimeSeriesHeader& override_header =
|
||||
cc->Options<TensorToMatrixCalculatorOptions>()
|
||||
.time_series_header_overrides();
|
||||
input_header->MergeFrom(override_header);
|
||||
CHECK(input_header->has_packet_rate())
|
||||
RET_CHECK(input_header->has_packet_rate())
|
||||
<< "The TimeSeriesHeader.packet_rate must be set.";
|
||||
if (!override_header.has_sample_rate()) {
|
||||
CHECK_EQ(input_header->num_samples(), 1)
|
||||
RET_CHECK_EQ(input_header->num_samples(), 1)
|
||||
<< "Currently the time series can only output single samples.";
|
||||
input_header->set_sample_rate(input_header->packet_rate());
|
||||
}
|
||||
@@ -186,20 +179,16 @@ absl::Status TensorToMatrixCalculator::Open(CalculatorContext* cc) {
|
||||
}
|
||||
|
||||
absl::Status TensorToMatrixCalculator::Process(CalculatorContext* cc) {
|
||||
// Daredevil requested CHECK for noisy failures rather than quieter RET_CHECK
|
||||
// failures. These are absolute conditions of the graph for the graph to be
|
||||
// valid, and if it is violated by any input anywhere, the graph will be
|
||||
// invalid for all inputs. A hard CHECK will enable faster debugging by
|
||||
// immediately exiting and more prominently displaying error messages.
|
||||
// Do not replace with RET_CHECKs.
|
||||
|
||||
// Verify that each reference stream packet corresponds to a tensor packet
|
||||
// otherwise the header information is invalid. If we don't have a reference
|
||||
// stream, Process() is only called when we have an input tensor and this is
|
||||
// always True.
|
||||
CHECK(cc->Inputs().HasTag(kTensor))
|
||||
RET_CHECK(cc->Inputs().HasTag(kTensor))
|
||||
<< "Tensor stream not available at same timestamp as the reference "
|
||||
"stream.";
|
||||
RET_CHECK(!cc->Inputs().Tag(kTensor).IsEmpty()) << "Tensor stream is empty.";
|
||||
RET_CHECK_OK(cc->Inputs().Tag(kTensor).Value().ValidateAsType<tf::Tensor>())
|
||||
<< "Tensor stream packet does not contain a Tensor.";
|
||||
|
||||
const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get<tf::Tensor>();
|
||||
CHECK(1 == input_tensor.dims() || 2 == input_tensor.dims())
|
||||
@@ -207,13 +196,12 @@ absl::Status TensorToMatrixCalculator::Process(CalculatorContext* cc) {
|
||||
const int32 length = input_tensor.dim_size(input_tensor.dims() - 1);
|
||||
const int32 width = (1 == input_tensor.dims()) ? 1 : input_tensor.dim_size(0);
|
||||
if (header_.has_num_channels()) {
|
||||
CHECK_EQ(length, header_.num_channels())
|
||||
RET_CHECK_EQ(length, header_.num_channels())
|
||||
<< "The number of channels at runtime does not match the header.";
|
||||
}
|
||||
if (header_.has_num_samples()) {
|
||||
CHECK_EQ(width, header_.num_samples())
|
||||
RET_CHECK_EQ(width, header_.num_samples())
|
||||
<< "The number of samples at runtime does not match the header.";
|
||||
;
|
||||
}
|
||||
auto output = absl::make_unique<Matrix>(width, length);
|
||||
*output =
|
||||
|
||||
@@ -98,388 +98,543 @@ class InferenceState {
|
||||
|
||||
// This calculator performs inference on a trained TensorFlow model.
|
||||
//
|
||||
// A mediapipe::TensorFlowSession with a model loaded and ready for use.
|
||||
// For this calculator it must include a tag_to_tensor_map.
|
||||
cc->InputSidePackets().Tag("SESSION").Set<TensorFlowSession>();
|
||||
if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS")) {
|
||||
cc->InputSidePackets()
|
||||
.Tag("RECURRENT_INIT_TENSORS")
|
||||
.Set<std::unique_ptr<std::map<std::string, tf::Tensor>>>();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
// TensorFlow Sessions can be created from checkpoint paths, frozen models, or
|
||||
// the SavedModel system. See the TensorFlowSessionFrom* packet generators for
|
||||
// details. Each of these methods defines a mapping between MediaPipe streams
|
||||
// and TensorFlow tensors. All of this information is passed in as an
|
||||
// input_side_packet.
|
||||
//
|
||||
// The input and output streams are TensorFlow tensors labeled by tags. The tags
|
||||
// for the streams are matched to feeds and fetchs in a TensorFlow session using
|
||||
// a named_signature.generic_signature in the ModelManifest. The
|
||||
// generic_signature is used as key-value pairs between the MediaPipe tag and
|
||||
// the TensorFlow tensor. The signature_name in the options proto determines
|
||||
// which named_signature is used. The keys in the generic_signature must be
|
||||
// valid MediaPipe tags ([A-Z0-9_]*, no lowercase or special characters). All of
|
||||
// the tensors corresponding to tags in the signature for input_streams are fed
|
||||
// to the model and for output_streams the tensors are fetched from the model.
|
||||
//
|
||||
// Other calculators are used to convert data to and from tensors, this op only
|
||||
// handles the TensorFlow session and batching. Batching occurs by concatenating
|
||||
// input tensors along the 0th dimension across timestamps. If the 0th dimension
|
||||
// is not a batch dimension, this calculator will add a 0th dimension by
|
||||
// default. Setting add_batch_dim_to_tensors to false disables the dimension
|
||||
// addition. Once batch_size inputs have been provided, the batch will be run
|
||||
// and the output tensors sent out on the output streams with timestamps
|
||||
// corresponding to the input stream packets. Setting the batch_size to 1
|
||||
// completely disables batching, but is indepdent of add_batch_dim_to_tensors.
|
||||
//
|
||||
// The TensorFlowInferenceCalculator also support feeding states recurrently for
|
||||
// RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the
|
||||
// recurrent tensors. Initializing the recurrent state can be handled by the
|
||||
// GraphTensorsPacketGenerator.
|
||||
//
|
||||
// The calculator updates two Counters to report timing information:
|
||||
// --<name>-TotalTimeUsecs = Total time spent running inference (in usecs),
|
||||
// --<name>-TotalProcessedTimestamps = # of instances processed
|
||||
// (approximately batches processed * batch_size),
|
||||
// where <name> is replaced with CalculatorGraphConfig::Node::name() if it
|
||||
// exists, or with TensorFlowInferenceCalculator if the name is not set. The
|
||||
// name must be set for timing information to be instance-specific in graphs
|
||||
// with multiple TensorFlowInferenceCalculators.
|
||||
//
|
||||
// Example config:
|
||||
// packet_generator {
|
||||
// packet_generator: "TensorFlowSessionFromSavedModelGenerator"
|
||||
// output_side_packet: "tensorflow_session"
|
||||
// options {
|
||||
// [mediapipe.TensorFlowSessionFromSavedModelGeneratorOptions.ext]: {
|
||||
// saved_model_path: "/path/to/saved/model"
|
||||
// signature_name: "mediapipe"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// node {
|
||||
// calculator: "TensorFlowInferenceCalculator"
|
||||
// input_stream: "IMAGES:image_tensors_keyed_in_signature_by_tag"
|
||||
// input_stream: "AUDIO:audio_tensors_keyed_in_signature_by_tag"
|
||||
// output_stream: "LABELS:softmax_tensor_keyed_in_signature_by_tag"
|
||||
// input_side_packet: "SESSION:tensorflow_session"
|
||||
// }
|
||||
//
|
||||
// Where the input and output streams are treated as Packet<tf::Tensor> and
|
||||
// the mediapipe_signature has tensor bindings between "IMAGES", "AUDIO", and
|
||||
// "LABELS" and their respective tensors exported to /path/to/bundle. For an
|
||||
// example of how this model was exported, see
|
||||
// tensorflow_inference_test_graph_generator.py
|
||||
//
|
||||
// It is possible to use a GraphDef proto that was not exported by exporter (i.e
|
||||
// without MetaGraph with bindings). Such GraphDef could contain all of its
|
||||
// parameters in-lined (for example, it can be the output of freeze_graph.py).
|
||||
// To instantiate a TensorFlow model from a GraphDef file, replace the
|
||||
// packet_factory above with TensorFlowSessionFromFrozenGraphGenerator:
|
||||
//
|
||||
// packet_generator {
|
||||
// packet_generator: "TensorFlowSessionFromFrozenGraphGenerator"
|
||||
// output_side_packet: "SESSION:tensorflow_session"
|
||||
// options {
|
||||
// [mediapipe.TensorFlowSessionFromFrozenGraphGeneratorOptions.ext]: {
|
||||
// graph_proto_path: "[PATH]"
|
||||
// tag_to_tensor_names {
|
||||
// key: "JPG_STRING"
|
||||
// value: "input:0"
|
||||
// }
|
||||
// tag_to_tensor_names {
|
||||
// key: "SOFTMAX"
|
||||
// value: "softmax:0"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// It is also possible to use a GraphDef proto and checkpoint file that have not
|
||||
// been frozen. This can be used to load graphs directly as they have been
|
||||
// written from training. However, it is more brittle and you are encouraged to
|
||||
// use a one of the more perminent formats described above. To instantiate a
|
||||
// TensorFlow model from a GraphDef file and checkpoint, replace the
|
||||
// packet_factory above with TensorFlowSessionFromModelCheckpointGenerator:
|
||||
//
|
||||
// packet_generator {
|
||||
// packet_generator: "TensorFlowSessionFromModelCheckpointGenerator"
|
||||
// output_side_packet: "SESSION:tensorflow_session"
|
||||
// options {
|
||||
// [mediapipe.TensorFlowSessionFromModelCheckpointGeneratorOptions.ext]: {
|
||||
// graph_proto_path: "[PATH]"
|
||||
// model_options {
|
||||
// checkpoint_path: "[PATH2]"
|
||||
// }
|
||||
// tag_to_tensor_names {
|
||||
// key: "JPG_STRING"
|
||||
// value: "input:0"
|
||||
// }
|
||||
// tag_to_tensor_names {
|
||||
// key: "SOFTMAX"
|
||||
// value: "softmax:0"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
public:
|
||||
// Counters for recording timing information. The actual names have the value
|
||||
// of CalculatorGraphConfig::Node::name() prepended.
|
||||
static constexpr char kTotalUsecsCounterSuffix[] = "TotalTimeUsecs";
|
||||
static constexpr char kTotalProcessedTimestampsCounterSuffix[] =
|
||||
"TotalProcessedTimestamps";
|
||||
static constexpr char kTotalSessionRunsTimeUsecsCounterSuffix[] =
|
||||
"TotalSessionRunsTimeUsecs";
|
||||
static constexpr char kTotalNumSessionRunsCounterSuffix[] =
|
||||
"TotalNumSessionRuns";
|
||||
|
||||
std::unique_ptr<InferenceState> CreateInferenceState(CalculatorContext* cc)
|
||||
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
|
||||
std::unique_ptr<InferenceState> inference_state =
|
||||
absl::make_unique<InferenceState>();
|
||||
if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS") &&
|
||||
!cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS").IsEmpty()) {
|
||||
std::map<std::string, tf::Tensor>* init_tensor_map;
|
||||
init_tensor_map = GetFromUniquePtr<std::map<std::string, tf::Tensor>>(
|
||||
cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS"));
|
||||
for (const auto& p : *init_tensor_map) {
|
||||
inference_state->input_tensor_batches_[p.first].emplace_back(p.second);
|
||||
TensorFlowInferenceCalculator() : session_(nullptr) {
|
||||
clock_ = std::unique_ptr<mediapipe::Clock>(
|
||||
mediapipe::MonotonicClock::CreateSynchronizedMonotonicClock());
|
||||
}
|
||||
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
const auto& options = cc->Options<TensorFlowInferenceCalculatorOptions>();
|
||||
RET_CHECK(!cc->Inputs().GetTags().empty());
|
||||
for (const std::string& tag : cc->Inputs().GetTags()) {
|
||||
// The tensorflow::Tensor with the tag equal to the graph node. May
|
||||
// have a TimeSeriesHeader if all present TimeSeriesHeaders match.
|
||||
if (!options.batched_input()) {
|
||||
cc->Inputs().Tag(tag).Set<tf::Tensor>();
|
||||
} else {
|
||||
cc->Inputs().Tag(tag).Set<std::vector<mediapipe::Packet>>();
|
||||
}
|
||||
}
|
||||
}
|
||||
return inference_state;
|
||||
}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override {
|
||||
options_ = cc->Options<TensorFlowInferenceCalculatorOptions>();
|
||||
|
||||
RET_CHECK(cc->InputSidePackets().HasTag("SESSION"));
|
||||
session_ = cc->InputSidePackets()
|
||||
.Tag("SESSION")
|
||||
.Get<TensorFlowSession>()
|
||||
.session.get();
|
||||
tag_to_tensor_map_ = cc->InputSidePackets()
|
||||
.Tag("SESSION")
|
||||
.Get<TensorFlowSession>()
|
||||
.tag_to_tensor_map;
|
||||
|
||||
// Validate and store the recurrent tags
|
||||
RET_CHECK(options_.has_batch_size());
|
||||
RET_CHECK(options_.batch_size() == 1 || options_.recurrent_tag_pair().empty())
|
||||
<< "To use recurrent_tag_pairs, batch_size must be 1.";
|
||||
for (const auto& tag_pair : options_.recurrent_tag_pair()) {
|
||||
const std::vector<std::string> tags = absl::StrSplit(tag_pair, ':');
|
||||
RET_CHECK_EQ(tags.size(), 2)
|
||||
<< "recurrent_tag_pair must be a colon "
|
||||
"separated std::string with two components: "
|
||||
<< tag_pair;
|
||||
RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[0]))
|
||||
<< "Can't find tag '" << tags[0] << "' in signature "
|
||||
<< options_.signature_name();
|
||||
RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[1]))
|
||||
<< "Can't find tag '" << tags[1] << "' in signature "
|
||||
<< options_.signature_name();
|
||||
recurrent_feed_tags_.insert(tags[0]);
|
||||
recurrent_fetch_tags_to_feed_tags_[tags[1]] = tags[0];
|
||||
}
|
||||
|
||||
// Check that all tags are present in this signature bound to tensors.
|
||||
for (const std::string& tag : cc->Inputs().GetTags()) {
|
||||
RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag))
|
||||
<< "Can't find tag '" << tag << "' in signature "
|
||||
<< options_.signature_name();
|
||||
}
|
||||
for (const std::string& tag : cc->Outputs().GetTags()) {
|
||||
RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag))
|
||||
<< "Can't find tag '" << tag << "' in signature "
|
||||
<< options_.signature_name();
|
||||
}
|
||||
|
||||
{
|
||||
absl::WriterMutexLock l(&mutex_);
|
||||
inference_state_ = std::unique_ptr<InferenceState>();
|
||||
}
|
||||
|
||||
if (options_.batch_size() == 1 || options_.batched_input()) {
|
||||
cc->SetOffset(0);
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// Adds a batch dimension to the input tensor if specified in the calculator
|
||||
// options.
|
||||
absl::Status AddBatchDimension(tf::Tensor* input_tensor) {
|
||||
if (options_.add_batch_dim_to_tensors()) {
|
||||
tf::TensorShape new_shape(input_tensor->shape());
|
||||
new_shape.InsertDim(0, 1);
|
||||
RET_CHECK(input_tensor->CopyFrom(*input_tensor, new_shape))
|
||||
<< "Could not add 0th dimension to tensor without changing its shape."
|
||||
<< " Current shape: " << input_tensor->shape().DebugString();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status AggregateTensorPacket(
|
||||
const std::string& tag_name, const Packet& packet,
|
||||
std::map<Timestamp, std::map<std::string, tf::Tensor>>*
|
||||
input_tensors_by_tag_by_timestamp,
|
||||
InferenceState* inference_state) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
|
||||
tf::Tensor input_tensor(packet.Get<tf::Tensor>());
|
||||
RET_CHECK_OK(AddBatchDimension(&input_tensor));
|
||||
if (mediapipe::ContainsKey(recurrent_feed_tags_, tag_name)) {
|
||||
// If we receive an input on a recurrent tag, override the state.
|
||||
// It's OK to override the global state because there is just one
|
||||
// input stream allowed for recurrent tensors.
|
||||
inference_state_->input_tensor_batches_[tag_name].clear();
|
||||
}
|
||||
(*input_tensors_by_tag_by_timestamp)[packet.Timestamp()].insert(
|
||||
std::make_pair(tag_name, input_tensor));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// Removes the batch dimension of the output tensor if specified in the
|
||||
// calculator options.
|
||||
absl::Status RemoveBatchDimension(tf::Tensor* output_tensor) {
|
||||
if (options_.add_batch_dim_to_tensors()) {
|
||||
tf::TensorShape new_shape(output_tensor->shape());
|
||||
new_shape.RemoveDim(0);
|
||||
RET_CHECK(output_tensor->CopyFrom(*output_tensor, new_shape))
|
||||
<< "Could not remove 0th dimension from tensor without changing its "
|
||||
<< "shape. Current shape: " << output_tensor->shape().DebugString()
|
||||
<< " (The expected first dimension is 1 for a batch element.)";
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
std::unique_ptr<InferenceState> inference_state_to_process;
|
||||
{
|
||||
absl::WriterMutexLock l(&mutex_);
|
||||
if (inference_state_ == nullptr) {
|
||||
inference_state_ = CreateInferenceState(cc);
|
||||
RET_CHECK(!cc->Outputs().GetTags().empty());
|
||||
for (const std::string& tag : cc->Outputs().GetTags()) {
|
||||
// The tensorflow::Tensor with tag equal to the graph node to
|
||||
// output. Any TimeSeriesHeader from the inputs will be forwarded
|
||||
// with channels set to 0.
|
||||
cc->Outputs().Tag(tag).Set<tf::Tensor>();
|
||||
}
|
||||
std::map<Timestamp, std::map<std::string, tf::Tensor>>
|
||||
input_tensors_by_tag_by_timestamp;
|
||||
for (const std::string& tag_as_node_name : cc->Inputs().GetTags()) {
|
||||
if (cc->Inputs().Tag(tag_as_node_name).IsEmpty()) {
|
||||
// Recurrent tensors can be empty.
|
||||
if (!mediapipe::ContainsKey(recurrent_feed_tags_, tag_as_node_name)) {
|
||||
if (options_.skip_on_missing_features()) {
|
||||
return absl::OkStatus();
|
||||
} else {
|
||||
return absl::InvalidArgumentError(absl::StrCat(
|
||||
"Tag ", tag_as_node_name,
|
||||
" not present at timestamp: ", cc->InputTimestamp().Value()));
|
||||
// A mediapipe::TensorFlowSession with a model loaded and ready for use.
|
||||
// For this calculator it must include a tag_to_tensor_map.
|
||||
cc->InputSidePackets().Tag("SESSION").Set<TensorFlowSession>();
|
||||
if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS")) {
|
||||
cc->InputSidePackets()
|
||||
.Tag("RECURRENT_INIT_TENSORS")
|
||||
.Set<std::unique_ptr<std::map<std::string, tf::Tensor>>>();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
std::unique_ptr<InferenceState> CreateInferenceState(CalculatorContext* cc)
|
||||
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
|
||||
std::unique_ptr<InferenceState> inference_state =
|
||||
absl::make_unique<InferenceState>();
|
||||
if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS") &&
|
||||
!cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS").IsEmpty()) {
|
||||
std::map<std::string, tf::Tensor>* init_tensor_map;
|
||||
init_tensor_map = GetFromUniquePtr<std::map<std::string, tf::Tensor>>(
|
||||
cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS"));
|
||||
for (const auto& p : *init_tensor_map) {
|
||||
inference_state->input_tensor_batches_[p.first].emplace_back(p.second);
|
||||
}
|
||||
}
|
||||
return inference_state;
|
||||
}
|
||||
|
||||
absl::Status Open(CalculatorContext* cc) override {
|
||||
options_ = cc->Options<TensorFlowInferenceCalculatorOptions>();
|
||||
|
||||
RET_CHECK(cc->InputSidePackets().HasTag("SESSION"));
|
||||
session_ = cc->InputSidePackets()
|
||||
.Tag("SESSION")
|
||||
.Get<TensorFlowSession>()
|
||||
.session.get();
|
||||
tag_to_tensor_map_ = cc->InputSidePackets()
|
||||
.Tag("SESSION")
|
||||
.Get<TensorFlowSession>()
|
||||
.tag_to_tensor_map;
|
||||
|
||||
// Validate and store the recurrent tags
|
||||
RET_CHECK(options_.has_batch_size());
|
||||
RET_CHECK(options_.batch_size() == 1 ||
|
||||
options_.recurrent_tag_pair().empty())
|
||||
<< "To use recurrent_tag_pairs, batch_size must be 1.";
|
||||
for (const auto& tag_pair : options_.recurrent_tag_pair()) {
|
||||
const std::vector<std::string> tags = absl::StrSplit(tag_pair, ':');
|
||||
RET_CHECK_EQ(tags.size(), 2)
|
||||
<< "recurrent_tag_pair must be a colon "
|
||||
"separated std::string with two components: "
|
||||
<< tag_pair;
|
||||
RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[0]))
|
||||
<< "Can't find tag '" << tags[0] << "' in signature "
|
||||
<< options_.signature_name();
|
||||
RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[1]))
|
||||
<< "Can't find tag '" << tags[1] << "' in signature "
|
||||
<< options_.signature_name();
|
||||
recurrent_feed_tags_.insert(tags[0]);
|
||||
recurrent_fetch_tags_to_feed_tags_[tags[1]] = tags[0];
|
||||
}
|
||||
|
||||
// Check that all tags are present in this signature bound to tensors.
|
||||
for (const std::string& tag : cc->Inputs().GetTags()) {
|
||||
RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag))
|
||||
<< "Can't find tag '" << tag << "' in signature "
|
||||
<< options_.signature_name();
|
||||
}
|
||||
for (const std::string& tag : cc->Outputs().GetTags()) {
|
||||
RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag))
|
||||
<< "Can't find tag '" << tag << "' in signature "
|
||||
<< options_.signature_name();
|
||||
}
|
||||
|
||||
{
|
||||
absl::WriterMutexLock l(&mutex_);
|
||||
inference_state_ = std::unique_ptr<InferenceState>();
|
||||
}
|
||||
|
||||
if (options_.batch_size() == 1 || options_.batched_input()) {
|
||||
cc->SetOffset(0);
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// Adds a batch dimension to the input tensor if specified in the calculator
|
||||
// options.
|
||||
absl::Status AddBatchDimension(tf::Tensor* input_tensor) {
|
||||
if (options_.add_batch_dim_to_tensors()) {
|
||||
tf::TensorShape new_shape(input_tensor->shape());
|
||||
new_shape.InsertDim(0, 1);
|
||||
RET_CHECK(input_tensor->CopyFrom(*input_tensor, new_shape))
|
||||
<< "Could not add 0th dimension to tensor without changing its shape."
|
||||
<< " Current shape: " << input_tensor->shape().DebugString();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status AggregateTensorPacket(
|
||||
const std::string& tag_name, const Packet& packet,
|
||||
std::map<Timestamp, std::map<std::string, tf::Tensor>>*
|
||||
input_tensors_by_tag_by_timestamp,
|
||||
InferenceState* inference_state) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
|
||||
tf::Tensor input_tensor(packet.Get<tf::Tensor>());
|
||||
RET_CHECK_OK(AddBatchDimension(&input_tensor));
|
||||
if (mediapipe::ContainsKey(recurrent_feed_tags_, tag_name)) {
|
||||
// If we receive an input on a recurrent tag, override the state.
|
||||
// It's OK to override the global state because there is just one
|
||||
// input stream allowed for recurrent tensors.
|
||||
inference_state_->input_tensor_batches_[tag_name].clear();
|
||||
}
|
||||
(*input_tensors_by_tag_by_timestamp)[packet.Timestamp()].insert(
|
||||
std::make_pair(tag_name, input_tensor));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// Removes the batch dimension of the output tensor if specified in the
|
||||
// calculator options.
|
||||
absl::Status RemoveBatchDimension(tf::Tensor* output_tensor) {
|
||||
if (options_.add_batch_dim_to_tensors()) {
|
||||
tf::TensorShape new_shape(output_tensor->shape());
|
||||
new_shape.RemoveDim(0);
|
||||
RET_CHECK(output_tensor->CopyFrom(*output_tensor, new_shape))
|
||||
<< "Could not remove 0th dimension from tensor without changing its "
|
||||
<< "shape. Current shape: " << output_tensor->shape().DebugString()
|
||||
<< " (The expected first dimension is 1 for a batch element.)";
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
std::unique_ptr<InferenceState> inference_state_to_process;
|
||||
{
|
||||
absl::WriterMutexLock l(&mutex_);
|
||||
if (inference_state_ == nullptr) {
|
||||
inference_state_ = CreateInferenceState(cc);
|
||||
}
|
||||
std::map<Timestamp, std::map<std::string, tf::Tensor>>
|
||||
input_tensors_by_tag_by_timestamp;
|
||||
for (const std::string& tag_as_node_name : cc->Inputs().GetTags()) {
|
||||
if (cc->Inputs().Tag(tag_as_node_name).IsEmpty()) {
|
||||
// Recurrent tensors can be empty.
|
||||
if (!mediapipe::ContainsKey(recurrent_feed_tags_, tag_as_node_name)) {
|
||||
if (options_.skip_on_missing_features()) {
|
||||
return absl::OkStatus();
|
||||
} else {
|
||||
return absl::InvalidArgumentError(absl::StrCat(
|
||||
"Tag ", tag_as_node_name,
|
||||
" not present at timestamp: ", cc->InputTimestamp().Value()));
|
||||
}
|
||||
}
|
||||
} else if (options_.batched_input()) {
|
||||
const auto& tensor_packets =
|
||||
cc->Inputs().Tag(tag_as_node_name).Get<std::vector<Packet>>();
|
||||
if (tensor_packets.size() > options_.batch_size()) {
|
||||
return absl::InvalidArgumentError(absl::StrCat(
|
||||
"Batch for tag ", tag_as_node_name,
|
||||
" has more packets than batch capacity. batch_size: ",
|
||||
options_.batch_size(), " packets: ", tensor_packets.size()));
|
||||
}
|
||||
for (const auto& packet : tensor_packets) {
|
||||
RET_CHECK_OK(AggregateTensorPacket(
|
||||
tag_as_node_name, packet, &input_tensors_by_tag_by_timestamp,
|
||||
inference_state_.get()));
|
||||
}
|
||||
} else {
|
||||
RET_CHECK_OK(AggregateTensorPacket(
|
||||
tag_as_node_name, cc->Inputs().Tag(tag_as_node_name).Value(),
|
||||
&input_tensors_by_tag_by_timestamp, inference_state_.get()));
|
||||
}
|
||||
} else if (options_.batched_input()) {
|
||||
const auto& tensor_packets =
|
||||
cc->Inputs().Tag(tag_as_node_name).Get<std::vector<Packet>>();
|
||||
if (tensor_packets.size() > options_.batch_size()) {
|
||||
return absl::InvalidArgumentError(absl::StrCat(
|
||||
"Batch for tag ", tag_as_node_name,
|
||||
" has more packets than batch capacity. batch_size: ",
|
||||
options_.batch_size(), " packets: ", tensor_packets.size()));
|
||||
}
|
||||
for (const auto& timestamp_and_input_tensors_by_tag :
|
||||
input_tensors_by_tag_by_timestamp) {
|
||||
inference_state_->batch_timestamps_.emplace_back(
|
||||
timestamp_and_input_tensors_by_tag.first);
|
||||
for (const auto& input_tensor_and_tag :
|
||||
timestamp_and_input_tensors_by_tag.second) {
|
||||
inference_state_->input_tensor_batches_[input_tensor_and_tag.first]
|
||||
.emplace_back(input_tensor_and_tag.second);
|
||||
}
|
||||
for (const auto& packet : tensor_packets) {
|
||||
RET_CHECK_OK(AggregateTensorPacket(tag_as_node_name, packet,
|
||||
&input_tensors_by_tag_by_timestamp,
|
||||
inference_state_.get()));
|
||||
}
|
||||
if (inference_state_->batch_timestamps_.size() == options_.batch_size() ||
|
||||
options_.batched_input()) {
|
||||
inference_state_to_process = std::move(inference_state_);
|
||||
inference_state_ = std::unique_ptr<InferenceState>();
|
||||
}
|
||||
}
|
||||
|
||||
if (inference_state_to_process) {
|
||||
MP_RETURN_IF_ERROR(
|
||||
OutputBatch(cc, std::move(inference_state_to_process)));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Close(CalculatorContext* cc) override {
|
||||
std::unique_ptr<InferenceState> inference_state_to_process = nullptr;
|
||||
{
|
||||
absl::WriterMutexLock l(&mutex_);
|
||||
if (cc->GraphStatus().ok() && inference_state_ != nullptr &&
|
||||
!inference_state_->batch_timestamps_.empty()) {
|
||||
inference_state_to_process = std::move(inference_state_);
|
||||
inference_state_ = std::unique_ptr<InferenceState>();
|
||||
}
|
||||
}
|
||||
if (inference_state_to_process) {
|
||||
MP_RETURN_IF_ERROR(
|
||||
OutputBatch(cc, std::move(inference_state_to_process)));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// When a batch of input tensors is ready to be run, runs TensorFlow and
|
||||
// outputs the output tensors. The output tensors have timestamps matching
|
||||
// the input tensor that formed that batch element. Any requested
|
||||
// batch_dimension is added and removed. This code takes advantage of the fact
|
||||
// that copying a tensor shares the same reference-counted, heap allocated
|
||||
// memory buffer. Therefore, copies are cheap and should not cause the memory
|
||||
// buffer to fall out of scope. In contrast, concat is only used where
|
||||
// necessary.
|
||||
absl::Status OutputBatch(CalculatorContext* cc,
|
||||
std::unique_ptr<InferenceState> inference_state) {
|
||||
const int64 start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
std::vector<std::pair<mediapipe::ProtoString, tf::Tensor>> input_tensors;
|
||||
|
||||
for (auto& keyed_tensors : inference_state->input_tensor_batches_) {
|
||||
if (options_.batch_size() == 1) {
|
||||
// Short circuit to avoid the cost of deep copying tensors in concat.
|
||||
if (!keyed_tensors.second.empty()) {
|
||||
input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first],
|
||||
keyed_tensors.second[0]);
|
||||
} else {
|
||||
// The input buffer can be empty for recurrent tensors.
|
||||
RET_CHECK(
|
||||
mediapipe::ContainsKey(recurrent_feed_tags_, keyed_tensors.first))
|
||||
<< "A non-recurrent tensor does not have an input: "
|
||||
<< keyed_tensors.first;
|
||||
}
|
||||
} else {
|
||||
RET_CHECK_OK(AggregateTensorPacket(
|
||||
tag_as_node_name, cc->Inputs().Tag(tag_as_node_name).Value(),
|
||||
&input_tensors_by_tag_by_timestamp, inference_state_.get()));
|
||||
}
|
||||
}
|
||||
for (const auto& timestamp_and_input_tensors_by_tag :
|
||||
input_tensors_by_tag_by_timestamp) {
|
||||
inference_state_->batch_timestamps_.emplace_back(
|
||||
timestamp_and_input_tensors_by_tag.first);
|
||||
for (const auto& input_tensor_and_tag :
|
||||
timestamp_and_input_tensors_by_tag.second) {
|
||||
inference_state_->input_tensor_batches_[input_tensor_and_tag.first]
|
||||
.emplace_back(input_tensor_and_tag.second);
|
||||
}
|
||||
}
|
||||
if (inference_state_->batch_timestamps_.size() == options_.batch_size() ||
|
||||
options_.batched_input()) {
|
||||
inference_state_to_process = std::move(inference_state_);
|
||||
inference_state_ = std::unique_ptr<InferenceState>();
|
||||
}
|
||||
}
|
||||
|
||||
if (inference_state_to_process) {
|
||||
MP_RETURN_IF_ERROR(OutputBatch(cc, std::move(inference_state_to_process)));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Close(CalculatorContext* cc) override {
|
||||
std::unique_ptr<InferenceState> inference_state_to_process = nullptr;
|
||||
{
|
||||
absl::WriterMutexLock l(&mutex_);
|
||||
if (cc->GraphStatus().ok() && inference_state_ != nullptr &&
|
||||
!inference_state_->batch_timestamps_.empty()) {
|
||||
inference_state_to_process = std::move(inference_state_);
|
||||
inference_state_ = std::unique_ptr<InferenceState>();
|
||||
}
|
||||
}
|
||||
if (inference_state_to_process) {
|
||||
MP_RETURN_IF_ERROR(OutputBatch(cc, std::move(inference_state_to_process)));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// When a batch of input tensors is ready to be run, runs TensorFlow and
|
||||
// outputs the output tensors. The output tensors have timestamps matching
|
||||
// the input tensor that formed that batch element. Any requested
|
||||
// batch_dimension is added and removed. This code takes advantage of the fact
|
||||
// that copying a tensor shares the same reference-counted, heap allocated
|
||||
// memory buffer. Therefore, copies are cheap and should not cause the memory
|
||||
// buffer to fall out of scope. In contrast, concat is only used where
|
||||
// necessary.
|
||||
absl::Status OutputBatch(CalculatorContext* cc,
|
||||
std::unique_ptr<InferenceState> inference_state) {
|
||||
const int64 start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
std::vector<std::pair<mediapipe::ProtoString, tf::Tensor>> input_tensors;
|
||||
|
||||
for (auto& keyed_tensors : inference_state->input_tensor_batches_) {
|
||||
if (options_.batch_size() == 1) {
|
||||
// Short circuit to avoid the cost of deep copying tensors in concat.
|
||||
if (!keyed_tensors.second.empty()) {
|
||||
// Pad by replicating the first tens or, then ignore the values.
|
||||
keyed_tensors.second.resize(options_.batch_size());
|
||||
std::fill(keyed_tensors.second.begin() +
|
||||
inference_state->batch_timestamps_.size(),
|
||||
keyed_tensors.second.end(), keyed_tensors.second[0]);
|
||||
tf::Tensor concated;
|
||||
const tf::Status concat_status =
|
||||
tf::tensor::Concat(keyed_tensors.second, &concated);
|
||||
CHECK(concat_status.ok()) << concat_status.ToString();
|
||||
input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first],
|
||||
keyed_tensors.second[0]);
|
||||
} else {
|
||||
// The input buffer can be empty for recurrent tensors.
|
||||
RET_CHECK(
|
||||
mediapipe::ContainsKey(recurrent_feed_tags_, keyed_tensors.first))
|
||||
<< "A non-recurrent tensor does not have an input: "
|
||||
<< keyed_tensors.first;
|
||||
concated);
|
||||
}
|
||||
} else {
|
||||
// Pad by replicating the first tens or, then ignore the values.
|
||||
keyed_tensors.second.resize(options_.batch_size());
|
||||
std::fill(keyed_tensors.second.begin() +
|
||||
inference_state->batch_timestamps_.size(),
|
||||
keyed_tensors.second.end(), keyed_tensors.second[0]);
|
||||
tf::Tensor concated;
|
||||
const tf::Status concat_status =
|
||||
tf::tensor::Concat(keyed_tensors.second, &concated);
|
||||
CHECK(concat_status.ok()) << concat_status.ToString();
|
||||
input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first],
|
||||
concated);
|
||||
}
|
||||
}
|
||||
inference_state->input_tensor_batches_.clear();
|
||||
std::vector<mediapipe::ProtoString> output_tensor_names;
|
||||
std::vector<std::string> output_name_in_signature;
|
||||
for (const std::string& tag : cc->Outputs().GetTags()) {
|
||||
output_tensor_names.emplace_back(tag_to_tensor_map_[tag]);
|
||||
output_name_in_signature.emplace_back(tag);
|
||||
}
|
||||
for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) {
|
||||
// Ensure that we always fetch the recurrent state tensors.
|
||||
if (std::find(output_name_in_signature.begin(),
|
||||
output_name_in_signature.end(),
|
||||
tag_pair.first) == output_name_in_signature.end()) {
|
||||
output_tensor_names.emplace_back(tag_to_tensor_map_[tag_pair.first]);
|
||||
output_name_in_signature.emplace_back(tag_pair.first);
|
||||
inference_state->input_tensor_batches_.clear();
|
||||
std::vector<mediapipe::ProtoString> output_tensor_names;
|
||||
std::vector<std::string> output_name_in_signature;
|
||||
for (const std::string& tag : cc->Outputs().GetTags()) {
|
||||
output_tensor_names.emplace_back(tag_to_tensor_map_[tag]);
|
||||
output_name_in_signature.emplace_back(tag);
|
||||
}
|
||||
}
|
||||
std::vector<tf::Tensor> outputs;
|
||||
for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) {
|
||||
// Ensure that we always fetch the recurrent state tensors.
|
||||
if (std::find(output_name_in_signature.begin(),
|
||||
output_name_in_signature.end(),
|
||||
tag_pair.first) == output_name_in_signature.end()) {
|
||||
output_tensor_names.emplace_back(tag_to_tensor_map_[tag_pair.first]);
|
||||
output_name_in_signature.emplace_back(tag_pair.first);
|
||||
}
|
||||
}
|
||||
std::vector<tf::Tensor> outputs;
|
||||
|
||||
SimpleSemaphore* session_run_throttle = nullptr;
|
||||
if (options_.max_concurrent_session_runs() > 0) {
|
||||
session_run_throttle =
|
||||
get_session_run_throttle(options_.max_concurrent_session_runs());
|
||||
session_run_throttle->Acquire(1);
|
||||
}
|
||||
const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
tf::Status tf_status;
|
||||
{
|
||||
SimpleSemaphore* session_run_throttle = nullptr;
|
||||
if (options_.max_concurrent_session_runs() > 0) {
|
||||
session_run_throttle =
|
||||
get_session_run_throttle(options_.max_concurrent_session_runs());
|
||||
session_run_throttle->Acquire(1);
|
||||
}
|
||||
const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
tf::Status tf_status;
|
||||
{
|
||||
#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
|
||||
tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
|
||||
tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
|
||||
#endif
|
||||
tf_status = session_->Run(input_tensors, output_tensor_names,
|
||||
{} /* target_node_names */, &outputs);
|
||||
}
|
||||
tf_status = session_->Run(input_tensors, output_tensor_names,
|
||||
{} /* target_node_names */, &outputs);
|
||||
}
|
||||
|
||||
if (session_run_throttle != nullptr) {
|
||||
session_run_throttle->Release(1);
|
||||
}
|
||||
if (session_run_throttle != nullptr) {
|
||||
session_run_throttle->Release(1);
|
||||
}
|
||||
|
||||
// RET_CHECK on the tf::Status object itself in order to print an
|
||||
// informative error message.
|
||||
RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
|
||||
// RET_CHECK on the tf::Status object itself in order to print an
|
||||
// informative error message.
|
||||
RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
|
||||
|
||||
const int64 run_end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
cc->GetCounter(kTotalSessionRunsTimeUsecsCounterSuffix)
|
||||
->IncrementBy(run_end_time - run_start_time);
|
||||
cc->GetCounter(kTotalNumSessionRunsCounterSuffix)->Increment();
|
||||
const int64 run_end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
cc->GetCounter(kTotalSessionRunsTimeUsecsCounterSuffix)
|
||||
->IncrementBy(run_end_time - run_start_time);
|
||||
cc->GetCounter(kTotalNumSessionRunsCounterSuffix)->Increment();
|
||||
|
||||
// Feed back the recurrent state.
|
||||
for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) {
|
||||
int pos = std::find(output_name_in_signature.begin(),
|
||||
output_name_in_signature.end(), tag_pair.first) -
|
||||
output_name_in_signature.begin();
|
||||
inference_state->input_tensor_batches_[tag_pair.second].emplace_back(
|
||||
outputs[pos]);
|
||||
}
|
||||
// Feed back the recurrent state.
|
||||
for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) {
|
||||
int pos = std::find(output_name_in_signature.begin(),
|
||||
output_name_in_signature.end(), tag_pair.first) -
|
||||
output_name_in_signature.begin();
|
||||
inference_state->input_tensor_batches_[tag_pair.second].emplace_back(
|
||||
outputs[pos]);
|
||||
}
|
||||
|
||||
absl::WriterMutexLock l(&mutex_);
|
||||
// Set that we want to split on each index of the 0th dimension.
|
||||
std::vector<tf::int64> split_vector(options_.batch_size(), 1);
|
||||
for (int i = 0; i < output_tensor_names.size(); ++i) {
|
||||
if (options_.batch_size() == 1) {
|
||||
if (cc->Outputs().HasTag(output_name_in_signature[i])) {
|
||||
tf::Tensor output_tensor(outputs[i]);
|
||||
RET_CHECK_OK(RemoveBatchDimension(&output_tensor));
|
||||
cc->Outputs()
|
||||
.Tag(output_name_in_signature[i])
|
||||
.Add(new tf::Tensor(output_tensor),
|
||||
inference_state->batch_timestamps_[0]);
|
||||
}
|
||||
} else {
|
||||
std::vector<tf::Tensor> split_tensors;
|
||||
const tf::Status split_status =
|
||||
tf::tensor::Split(outputs[i], split_vector, &split_tensors);
|
||||
CHECK(split_status.ok()) << split_status.ToString();
|
||||
// Loop over timestamps so that we don't copy the padding.
|
||||
for (int j = 0; j < inference_state->batch_timestamps_.size(); ++j) {
|
||||
tf::Tensor output_tensor(split_tensors[j]);
|
||||
RET_CHECK_OK(RemoveBatchDimension(&output_tensor));
|
||||
cc->Outputs()
|
||||
.Tag(output_name_in_signature[i])
|
||||
.Add(new tf::Tensor(output_tensor),
|
||||
inference_state->batch_timestamps_[j]);
|
||||
absl::WriterMutexLock l(&mutex_);
|
||||
// Set that we want to split on each index of the 0th dimension.
|
||||
std::vector<tf::int64> split_vector(options_.batch_size(), 1);
|
||||
for (int i = 0; i < output_tensor_names.size(); ++i) {
|
||||
if (options_.batch_size() == 1) {
|
||||
if (cc->Outputs().HasTag(output_name_in_signature[i])) {
|
||||
tf::Tensor output_tensor(outputs[i]);
|
||||
RET_CHECK_OK(RemoveBatchDimension(&output_tensor));
|
||||
cc->Outputs()
|
||||
.Tag(output_name_in_signature[i])
|
||||
.Add(new tf::Tensor(output_tensor),
|
||||
inference_state->batch_timestamps_[0]);
|
||||
}
|
||||
} else {
|
||||
std::vector<tf::Tensor> split_tensors;
|
||||
const tf::Status split_status =
|
||||
tf::tensor::Split(outputs[i], split_vector, &split_tensors);
|
||||
CHECK(split_status.ok()) << split_status.ToString();
|
||||
// Loop over timestamps so that we don't copy the padding.
|
||||
for (int j = 0; j < inference_state->batch_timestamps_.size(); ++j) {
|
||||
tf::Tensor output_tensor(split_tensors[j]);
|
||||
RET_CHECK_OK(RemoveBatchDimension(&output_tensor));
|
||||
cc->Outputs()
|
||||
.Tag(output_name_in_signature[i])
|
||||
.Add(new tf::Tensor(output_tensor),
|
||||
inference_state->batch_timestamps_[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get end time and report.
|
||||
const int64 end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
cc->GetCounter(kTotalUsecsCounterSuffix)
|
||||
->IncrementBy(end_time - start_time);
|
||||
cc->GetCounter(kTotalProcessedTimestampsCounterSuffix)
|
||||
->IncrementBy(inference_state->batch_timestamps_.size());
|
||||
|
||||
// Make sure we hold on to the recursive state.
|
||||
if (!options_.recurrent_tag_pair().empty()) {
|
||||
inference_state_ = std::move(inference_state);
|
||||
inference_state_->batch_timestamps_.clear();
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// Get end time and report.
|
||||
const int64 end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
cc->GetCounter(kTotalUsecsCounterSuffix)->IncrementBy(end_time - start_time);
|
||||
cc->GetCounter(kTotalProcessedTimestampsCounterSuffix)
|
||||
->IncrementBy(inference_state->batch_timestamps_.size());
|
||||
private:
|
||||
// The Session object is provided by a packet factory and is owned by the
|
||||
// MediaPipe framework. Individual calls are thread-safe, but session state
|
||||
// may be shared across threads.
|
||||
tf::Session* session_;
|
||||
|
||||
// Make sure we hold on to the recursive state.
|
||||
if (!options_.recurrent_tag_pair().empty()) {
|
||||
inference_state_ = std::move(inference_state);
|
||||
inference_state_->batch_timestamps_.clear();
|
||||
// A mapping between stream tags and the tensor names they are bound to.
|
||||
std::map<std::string, std::string> tag_to_tensor_map_;
|
||||
|
||||
absl::Mutex mutex_;
|
||||
std::unique_ptr<InferenceState> inference_state_ ABSL_GUARDED_BY(mutex_);
|
||||
|
||||
// The options for the calculator.
|
||||
TensorFlowInferenceCalculatorOptions options_;
|
||||
|
||||
// Store the feed and fetch tags for feed/fetch recurrent networks.
|
||||
std::set<std::string> recurrent_feed_tags_;
|
||||
std::map<std::string, std::string> recurrent_fetch_tags_to_feed_tags_;
|
||||
|
||||
// Clock used to measure the computation time in OutputBatch().
|
||||
std::unique_ptr<mediapipe::Clock> clock_;
|
||||
|
||||
// The static singleton semaphore to throttle concurrent session runs.
|
||||
static SimpleSemaphore* get_session_run_throttle(
|
||||
int32 max_concurrent_session_runs) {
|
||||
static SimpleSemaphore* session_run_throttle =
|
||||
new SimpleSemaphore(max_concurrent_session_runs);
|
||||
return session_run_throttle;
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// The Session object is provided by a packet factory and is owned by the
|
||||
// MediaPipe framework. Individual calls are thread-safe, but session state may
|
||||
// be shared across threads.
|
||||
tf::Session* session_;
|
||||
|
||||
// A mapping between stream tags and the tensor names they are bound to.
|
||||
std::map<std::string, std::string> tag_to_tensor_map_;
|
||||
|
||||
absl::Mutex mutex_;
|
||||
std::unique_ptr<InferenceState> inference_state_ ABSL_GUARDED_BY(mutex_);
|
||||
|
||||
// The options for the calculator.
|
||||
TensorFlowInferenceCalculatorOptions options_;
|
||||
|
||||
// Store the feed and fetch tags for feed/fetch recurrent networks.
|
||||
std::set<std::string> recurrent_feed_tags_;
|
||||
std::map<std::string, std::string> recurrent_fetch_tags_to_feed_tags_;
|
||||
|
||||
// Clock used to measure the computation time in OutputBatch().
|
||||
std::unique_ptr<mediapipe::Clock> clock_;
|
||||
|
||||
// The static singleton semaphore to throttle concurrent session runs.
|
||||
static SimpleSemaphore* get_session_run_throttle(
|
||||
int32 max_concurrent_session_runs) {
|
||||
static SimpleSemaphore* session_run_throttle =
|
||||
new SimpleSemaphore(max_concurrent_session_runs);
|
||||
return session_run_throttle;
|
||||
}
|
||||
}
|
||||
;
|
||||
};
|
||||
REGISTER_CALCULATOR(TensorFlowInferenceCalculator);
|
||||
|
||||
constexpr char TensorFlowInferenceCalculator::kTotalUsecsCounterSuffix[];
|
||||
|
||||
@@ -16,12 +16,12 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
|
||||
+1
-1
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
|
||||
@@ -19,7 +20,6 @@
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/commandlineflags.h"
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user