Compare commits
958
Commits
@@ -87,6 +87,9 @@ build:ios_fat --config=ios
|
|||||||
build:ios_fat --ios_multi_cpus=armv7,arm64
|
build:ios_fat --ios_multi_cpus=armv7,arm64
|
||||||
build:ios_fat --watchos_cpus=armv7k
|
build:ios_fat --watchos_cpus=armv7k
|
||||||
|
|
||||||
|
build:ios_sim_fat --config=ios
|
||||||
|
build:ios_sim_fat --ios_multi_cpus=x86_64,sim_arm64
|
||||||
|
|
||||||
build:darwin_x86_64 --apple_platform_type=macos
|
build:darwin_x86_64 --apple_platform_type=macos
|
||||||
build:darwin_x86_64 --macos_minimum_os=10.12
|
build:darwin_x86_64 --macos_minimum_os=10.12
|
||||||
build:darwin_x86_64 --cpu=darwin_x86_64
|
build:darwin_x86_64 --cpu=darwin_x86_64
|
||||||
|
|||||||
+2
-4
@@ -40,18 +40,16 @@ body:
|
|||||||
label: Programming Language and version (e.g. C++, Python, Java)
|
label: Programming Language and version (e.g. C++, Python, Java)
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: input
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: input
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
+2
-4
@@ -41,18 +41,16 @@ body:
|
|||||||
label: Task name (e.g. Image classification, Gesture recognition etc.)
|
label: Task name (e.g. Image classification, Gesture recognition etc.)
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: input
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: input
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
+2
-4
@@ -31,18 +31,16 @@ body:
|
|||||||
label: URL that shows the problem
|
label: URL that shows the problem
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: input
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: input
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: textarea
|
||||||
+5
-9
@@ -28,37 +28,33 @@ body:
|
|||||||
- 'No'
|
- 'No'
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: input
|
||||||
id: behaviour
|
id: behaviour
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the feature and the current behaviour/state
|
label: Describe the feature and the current behaviour/state
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: input
|
||||||
id: api_change
|
id: api_change
|
||||||
attributes:
|
attributes:
|
||||||
label: Will this change the current API? How?
|
label: Will this change the current API? How?
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: input
|
||||||
id: benifit
|
id: benifit
|
||||||
attributes:
|
attributes:
|
||||||
label: Who will benefit with this feature?
|
label: Who will benefit with this feature?
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: input
|
||||||
id: use_case
|
id: use_case
|
||||||
attributes:
|
attributes:
|
||||||
label: Please specify the use cases for this feature
|
label: Please specify the use cases for this feature
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: input
|
||||||
id: info_other
|
id: info_other
|
||||||
attributes:
|
attributes:
|
||||||
label: Any Other info
|
label: Any Other info
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
+1
-2
@@ -87,14 +87,13 @@ body:
|
|||||||
placeholder:
|
placeholder:
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: input
|
||||||
id: what-happened
|
id: what-happened
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the problem
|
label: Describe the problem
|
||||||
description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html)
|
description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html)
|
||||||
placeholder: Tell us what you see!
|
placeholder: Tell us what you see!
|
||||||
value: "A bug happened!"
|
value: "A bug happened!"
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
+2
-4
@@ -80,18 +80,16 @@ body:
|
|||||||
label: Xcode & Tulsi version (if issue is related to building for iOS)
|
label: Xcode & Tulsi version (if issue is related to building for iOS)
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: input
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: input
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
+2
-4
@@ -48,18 +48,16 @@ body:
|
|||||||
placeholder: e.g. C++, Python, Java
|
placeholder: e.g. C++, Python, Java
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: input
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: input
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
render: shell
|
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: textarea
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
# Copyright 2021 The MediaPipe Authors.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
# ============================================================================
|
|
||||||
#
|
|
||||||
# This file was assembled from multiple pieces, whose use is documented
|
|
||||||
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
|
||||||
# for more information.
|
|
||||||
|
|
||||||
# Number of days of inactivity before an Issue or Pull Request becomes stale
|
|
||||||
daysUntilStale: 7
|
|
||||||
# Number of days of inactivity before a stale Issue or Pull Request is closed
|
|
||||||
daysUntilClose: 7
|
|
||||||
# Only issues or pull requests with all of these labels are checked if stale. Defaults to `[]` (disabled)
|
|
||||||
onlyLabels:
|
|
||||||
- stat:awaiting response
|
|
||||||
# Comment to post when marking as stale. Set to `false` to disable
|
|
||||||
markComment: >
|
|
||||||
This issue has been automatically marked as stale because it has not had
|
|
||||||
recent activity. It will be closed if no further activity occurs. Thank you.
|
|
||||||
# Comment to post when removing the stale label. Set to `false` to disable
|
|
||||||
unmarkComment: false
|
|
||||||
closeComment: >
|
|
||||||
Closing as stale. Please reopen if you'd like to work on this further.
|
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
|
# This workflow alerts and then closes the stale issues/PRs after specific time
|
||||||
|
# You can adjust the behavior by modifying this file.
|
||||||
|
# For more information, see:
|
||||||
|
# https://github.com/actions/stale
|
||||||
|
|
||||||
|
name: 'Close stale issues and PRs'
|
||||||
|
"on":
|
||||||
|
schedule:
|
||||||
|
- cron: "30 1 * * *"
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
issues: write
|
||||||
|
pull-requests: write
|
||||||
|
jobs:
|
||||||
|
stale:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: 'actions/stale@v7'
|
||||||
|
with:
|
||||||
|
# Comma separated list of labels that can be assigned to issues to exclude them from being marked as stale.
|
||||||
|
exempt-issue-labels: 'override-stale'
|
||||||
|
# Comma separated list of labels that can be assigned to PRs to exclude them from being marked as stale.
|
||||||
|
exempt-pr-labels: "override-stale"
|
||||||
|
# Limit the No. of API calls in one run default value is 30.
|
||||||
|
operations-per-run: 500
|
||||||
|
# Prevent to remove stale label when PRs or issues are updated.
|
||||||
|
remove-stale-when-updated: true
|
||||||
|
# List of labels to remove when issues/PRs unstale.
|
||||||
|
labels-to-remove-when-unstale: 'stat:awaiting response'
|
||||||
|
# comment on issue if not active for more then 7 days.
|
||||||
|
stale-issue-message: 'This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.'
|
||||||
|
# comment on PR if not active for more then 14 days.
|
||||||
|
stale-pr-message: 'This PR has been marked stale because it has no recent activity since 14 days. It will be closed if no further activity occurs. Thank you.'
|
||||||
|
# comment on issue if stale for more then 7 days.
|
||||||
|
close-issue-message: This issue was closed due to lack of activity after being marked stale for past 7 days.
|
||||||
|
# comment on PR if stale for more then 14 days.
|
||||||
|
close-pr-message: This PR was closed due to lack of activity after being marked stale for past 14 days.
|
||||||
|
# Number of days of inactivity before an Issue Request becomes stale
|
||||||
|
days-before-issue-stale: 7
|
||||||
|
# Number of days of inactivity before a stale Issue is closed
|
||||||
|
days-before-issue-close: 7
|
||||||
|
# reason for closed the issue default value is not_planned
|
||||||
|
close-issue-reason: completed
|
||||||
|
# Number of days of inactivity before a stale PR is closed
|
||||||
|
days-before-pr-close: 14
|
||||||
|
# Number of days of inactivity before an PR Request becomes stale
|
||||||
|
days-before-pr-stale: 14
|
||||||
|
# Check for label to stale or close the issue/PR
|
||||||
|
any-of-labels: 'stat:awaiting response'
|
||||||
|
# override stale to stalled for PR
|
||||||
|
stale-pr-label: 'stale'
|
||||||
|
# override stale to stalled for Issue
|
||||||
|
stale-issue-label: "stale"
|
||||||
@@ -1,99 +1,121 @@
|
|||||||
---
|
---
|
||||||
layout: default
|
layout: forward
|
||||||
|
target: https://developers.google.com/mediapipe
|
||||||
title: Home
|
title: Home
|
||||||
nav_order: 1
|
nav_order: 1
|
||||||
---
|
---
|
||||||
|
|
||||||

|
|
||||||
|
|
||||||
----
|
----
|
||||||
|
|
||||||
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
|
**Attention:** *We have moved to
|
||||||
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
|
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
|
||||||
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
||||||
|
|
||||||
*This notice and web page will be removed on June 1, 2023.*
|

|
||||||
|
|
||||||
----
|
**Attention**: MediaPipe Solutions Preview is an early release. [Learn
|
||||||
|
more](https://developers.google.com/mediapipe/solutions/about#notice).
|
||||||
|
|
||||||
<br><br><br><br><br><br><br><br><br><br>
|
**On-device machine learning for everyone**
|
||||||
<br><br><br><br><br><br><br><br><br><br>
|
|
||||||
<br><br><br><br><br><br><br><br><br><br>
|
|
||||||
|
|
||||||
--------------------------------------------------------------------------------
|
Delight your customers with innovative machine learning features. MediaPipe
|
||||||
|
contains everything that you need to customize and deploy to mobile (Android,
|
||||||
|
iOS), web, desktop, edge devices, and IoT, effortlessly.
|
||||||
|
|
||||||
## Live ML anywhere
|
* [See demos](https://goo.gle/mediapipe-studio)
|
||||||
|
* [Learn more](https://developers.google.com/mediapipe/solutions)
|
||||||
|
|
||||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
## Get started
|
||||||
ML solutions for live and streaming media.
|
|
||||||
|
|
||||||
 | 
|
You can get started with MediaPipe Solutions by by checking out any of the
|
||||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
developer guides for
|
||||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
|
||||||
 | 
|
[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
|
||||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
and
|
||||||
|
[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
|
||||||
|
tasks. If you need help setting up a development environment for use with
|
||||||
|
MediaPipe Tasks, check out the setup guides for
|
||||||
|
[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
|
||||||
|
apps](https://developers.google.com/mediapipe/solutions/setup_web), and
|
||||||
|
[Python](https://developers.google.com/mediapipe/solutions/setup_python).
|
||||||
|
|
||||||
----
|
## Solutions
|
||||||
|
|
||||||
## ML solutions in MediaPipe
|
MediaPipe Solutions provides a suite of libraries and tools for you to quickly
|
||||||
|
apply artificial intelligence (AI) and machine learning (ML) techniques in your
|
||||||
|
applications. You can plug these solutions into your applications immediately,
|
||||||
|
customize them to your needs, and use them across multiple development
|
||||||
|
platforms. MediaPipe Solutions is part of the MediaPipe [open source
|
||||||
|
project](https://github.com/google/mediapipe), so you can further customize the
|
||||||
|
solutions code to meet your application needs.
|
||||||
|
|
||||||
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
|
These libraries and resources provide the core functionality for each MediaPipe
|
||||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
|
Solution:
|
||||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
|
|
||||||
|
|
||||||
Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
|
||||||
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
solutions. [Learn
|
||||||
[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
more](https://developers.google.com/mediapipe/solutions/tasks).
|
||||||
|
* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
|
||||||
|
solution.
|
||||||
|
|
||||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
These tools let you customize and evaluate solutions:
|
||||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
|
||||||
|
|
||||||
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
* **MediaPipe Model Maker**: Customize models for solutions with your data.
|
||||||
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
|
[Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
|
||||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
|
* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
|
||||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
browser. [Learn
|
||||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
|
more](https://developers.google.com/mediapipe/solutions/studio).
|
||||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
||||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
||||||
[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
||||||
[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
||||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
|
|
||||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
|
||||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
|
||||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
|
||||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
|
|
||||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
|
||||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
|
||||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
|
||||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
|
||||||
|
|
||||||
See also
|
### Legacy solutions
|
||||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
|
||||||
for ML models released in MediaPipe.
|
|
||||||
|
|
||||||
## Getting started
|
We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||||
|
as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
|
||||||
|
a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||||
|
for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
|
||||||
|
and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
|
||||||
|
provided on an as-is basis.
|
||||||
|
|
||||||
To start using MediaPipe
|
For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
|
||||||
[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
|
|
||||||
lines code, see example code and demos in
|
|
||||||
[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
|
|
||||||
[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
|
|
||||||
|
|
||||||
To use MediaPipe in C++, Android and iOS, which allow further customization of
|
## Framework
|
||||||
the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
|
|
||||||
building your own, learn how to
|
|
||||||
[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
|
|
||||||
start building example applications in
|
|
||||||
[C++](https://google.github.io/mediapipe/getting_started/cpp),
|
|
||||||
[Android](https://google.github.io/mediapipe/getting_started/android) and
|
|
||||||
[iOS](https://google.github.io/mediapipe/getting_started/ios).
|
|
||||||
|
|
||||||
The source code is hosted in the
|
To start using MediaPipe Framework, [install MediaPipe
|
||||||
[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
|
Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
|
||||||
run code search using
|
and start building example applications in C++, Android, and iOS.
|
||||||
[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
|
|
||||||
|
|
||||||
## Publications
|
[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
|
||||||
|
low-level component used to build efficient on-device machine learning
|
||||||
|
pipelines, similar to the premade MediaPipe Solutions.
|
||||||
|
|
||||||
|
Before using MediaPipe Framework, familiarize yourself with the following key
|
||||||
|
[Framework
|
||||||
|
concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
|
||||||
|
|
||||||
|
* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
|
||||||
|
* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
|
||||||
|
* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
|
||||||
|
|
||||||
|
## Community
|
||||||
|
|
||||||
|
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
|
||||||
|
users.
|
||||||
|
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||||
|
community discussion around MediaPipe.
|
||||||
|
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||||
|
curated list of awesome MediaPipe related frameworks, libraries and
|
||||||
|
software.
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
We welcome contributions. Please follow these
|
||||||
|
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||||
|
|
||||||
|
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||||
|
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||||
|
|
||||||
|
## Resources
|
||||||
|
|
||||||
|
### Publications
|
||||||
|
|
||||||
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
||||||
in Google Developers Blog
|
in Google Developers Blog
|
||||||
@@ -102,7 +124,8 @@ run code search using
|
|||||||
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
||||||
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
||||||
in Google Developers Blog
|
in Google Developers Blog
|
||||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
|
||||||
|
Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||||
in Google AI Blog
|
in Google AI Blog
|
||||||
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
||||||
in Google AI Blog
|
in Google AI Blog
|
||||||
@@ -130,43 +153,6 @@ run code search using
|
|||||||
in Google AI Blog
|
in Google AI Blog
|
||||||
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
||||||
|
|
||||||
## Videos
|
### Videos
|
||||||
|
|
||||||
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
|
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
|
||||||
|
|
||||||
## Events
|
|
||||||
|
|
||||||
* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
|
|
||||||
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
|
|
||||||
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
|
|
||||||
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
|
|
||||||
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
|
|
||||||
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
|
|
||||||
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
|
|
||||||
Seoul, Korea ICCV
|
|
||||||
2019](https://research.google.com/youtube8m/workshop2019/index.html)
|
|
||||||
* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
|
|
||||||
* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
|
|
||||||
([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
|
|
||||||
* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
|
|
||||||
|
|
||||||
## Community
|
|
||||||
|
|
||||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
|
||||||
curated list of awesome MediaPipe related frameworks, libraries and software
|
|
||||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
|
|
||||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
|
||||||
community discussion around MediaPipe
|
|
||||||
|
|
||||||
## Alpha disclaimer
|
|
||||||
|
|
||||||
MediaPipe is currently in alpha at v0.7. We may be still making breaking API
|
|
||||||
changes and expect to get to stable APIs by v1.0.
|
|
||||||
|
|
||||||
## Contributing
|
|
||||||
|
|
||||||
We welcome contributions. Please follow these
|
|
||||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
|
||||||
|
|
||||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
|
||||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
|
||||||
|
|||||||
@@ -45,12 +45,13 @@ http_archive(
|
|||||||
)
|
)
|
||||||
|
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "rules_foreign_cc",
|
name = "rules_foreign_cc",
|
||||||
strip_prefix = "rules_foreign_cc-0.1.0",
|
sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
|
||||||
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
|
strip_prefix = "rules_foreign_cc-0.9.0",
|
||||||
|
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
|
||||||
)
|
)
|
||||||
|
|
||||||
load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
|
load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
|
||||||
|
|
||||||
rules_foreign_cc_dependencies()
|
rules_foreign_cc_dependencies()
|
||||||
|
|
||||||
@@ -72,12 +73,9 @@ http_archive(
|
|||||||
http_archive(
|
http_archive(
|
||||||
name = "zlib",
|
name = "zlib",
|
||||||
build_file = "@//third_party:zlib.BUILD",
|
build_file = "@//third_party:zlib.BUILD",
|
||||||
sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
|
sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
|
||||||
strip_prefix = "zlib-1.2.11",
|
strip_prefix = "zlib-1.2.13",
|
||||||
urls = [
|
url = "http://zlib.net/fossils/zlib-1.2.13.tar.gz",
|
||||||
"http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
|
|
||||||
"http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
|
|
||||||
],
|
|
||||||
patches = [
|
patches = [
|
||||||
"@//third_party:zlib.diff",
|
"@//third_party:zlib.diff",
|
||||||
],
|
],
|
||||||
@@ -156,22 +154,41 @@ http_archive(
|
|||||||
# 2020-08-21
|
# 2020-08-21
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "com_github_glog_glog",
|
name = "com_github_glog_glog",
|
||||||
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
|
strip_prefix = "glog-0.6.0",
|
||||||
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
|
sha256 = "8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6",
|
||||||
urls = [
|
urls = [
|
||||||
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
|
"https://github.com/google/glog/archive/v0.6.0.tar.gz",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "com_github_glog_glog_no_gflags",
|
name = "com_github_glog_glog_no_gflags",
|
||||||
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
|
strip_prefix = "glog-0.6.0",
|
||||||
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
|
sha256 = "8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6",
|
||||||
build_file = "@//third_party:glog_no_gflags.BUILD",
|
build_file = "@//third_party:glog_no_gflags.BUILD",
|
||||||
urls = [
|
urls = [
|
||||||
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
|
"https://github.com/google/glog/archive/v0.6.0.tar.gz",
|
||||||
],
|
],
|
||||||
patches = [
|
patches = [
|
||||||
"@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff",
|
"@//third_party:com_github_glog_glog.diff",
|
||||||
|
],
|
||||||
|
patch_args = [
|
||||||
|
"-p1",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2023-06-05
|
||||||
|
# This version of Glog is required for Windows support, but currently causes
|
||||||
|
# crashes on some Android devices.
|
||||||
|
http_archive(
|
||||||
|
name = "com_github_glog_glog_windows",
|
||||||
|
strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
|
||||||
|
sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
|
||||||
|
urls = [
|
||||||
|
"https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
|
||||||
|
],
|
||||||
|
patches = [
|
||||||
|
"@//third_party:com_github_glog_glog.diff",
|
||||||
|
"@//third_party:com_github_glog_glog_windows_patch.diff",
|
||||||
],
|
],
|
||||||
patch_args = [
|
patch_args = [
|
||||||
"-p1",
|
"-p1",
|
||||||
@@ -239,6 +256,16 @@ http_archive(
|
|||||||
repo_mapping = {"@com_google_glog" : "@com_github_glog_glog_no_gflags"},
|
repo_mapping = {"@com_google_glog" : "@com_github_glog_glog_no_gflags"},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
http_archive(
|
||||||
|
name = "darts_clone",
|
||||||
|
build_file = "@//third_party:darts_clone.BUILD",
|
||||||
|
sha256 = "c97f55d05c98da6fcaf7f9ecc6a6dc6bc5b18b8564465f77abff8879d446491c",
|
||||||
|
strip_prefix = "darts-clone-e40ce4627526985a7767444b6ed6893ab6ff8983",
|
||||||
|
urls = [
|
||||||
|
"https://github.com/s-yata/darts-clone/archive/e40ce4627526985a7767444b6ed6893ab6ff8983.zip",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "org_tensorflow_text",
|
name = "org_tensorflow_text",
|
||||||
sha256 = "f64647276f7288d1b1fe4c89581d51404d0ce4ae97f2bcc4c19bd667549adca8",
|
sha256 = "f64647276f7288d1b1fe4c89581d51404d0ce4ae97f2bcc4c19bd667549adca8",
|
||||||
@@ -256,10 +283,10 @@ http_archive(
|
|||||||
|
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "com_googlesource_code_re2",
|
name = "com_googlesource_code_re2",
|
||||||
sha256 = "e06b718c129f4019d6e7aa8b7631bee38d3d450dd980246bfaf493eb7db67868",
|
sha256 = "ef516fb84824a597c4d5d0d6d330daedb18363b5a99eda87d027e6bdd9cba299",
|
||||||
strip_prefix = "re2-fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8",
|
strip_prefix = "re2-03da4fc0857c285e3a26782f6bc8931c4c950df4",
|
||||||
urls = [
|
urls = [
|
||||||
"https://github.com/google/re2/archive/fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8.tar.gz",
|
"https://github.com/google/re2/archive/03da4fc0857c285e3a26782f6bc8931c4c950df4.tar.gz",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -365,6 +392,22 @@ http_archive(
|
|||||||
url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
|
url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Building an opencv.xcframework from the OpenCV 4.5.3 sources is necessary for
|
||||||
|
# MediaPipe iOS Task Libraries to be supported on arm64(M1) Macs. An
|
||||||
|
# `opencv.xcframework` archive has not been released and it is recommended to
|
||||||
|
# build the same from source using a script provided in OpenCV 4.5.0 upwards.
|
||||||
|
# OpenCV is fixed to version to 4.5.3 since swift support can only be disabled
|
||||||
|
# from 4.5.3 upwards. This is needed to avoid errors when the library is linked
|
||||||
|
# in Xcode. Swift support will be added in when the final binary MediaPipe iOS
|
||||||
|
# Task libraries are built.
|
||||||
|
http_archive(
|
||||||
|
name = "ios_opencv_source",
|
||||||
|
sha256 = "a61e7a4618d353140c857f25843f39b2abe5f451b018aab1604ef0bc34cd23d5",
|
||||||
|
build_file = "@//third_party:opencv_ios_source.BUILD",
|
||||||
|
type = "zip",
|
||||||
|
url = "https://github.com/opencv/opencv/archive/refs/tags/4.5.3.zip",
|
||||||
|
)
|
||||||
|
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "stblib",
|
name = "stblib",
|
||||||
strip_prefix = "stb-b42009b3b9d4ca35bc703f5310eedc74f584be58",
|
strip_prefix = "stb-b42009b3b9d4ca35bc703f5310eedc74f584be58",
|
||||||
@@ -458,9 +501,10 @@ http_archive(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# TensorFlow repo should always go after the other external dependencies.
|
# TensorFlow repo should always go after the other external dependencies.
|
||||||
# TF on 2023-03-08.
|
# TF on 2023-07-26.
|
||||||
_TENSORFLOW_GIT_COMMIT = "24f7ee636d62e1f8d8330357f8bbd65956dfb84d"
|
_TENSORFLOW_GIT_COMMIT = "e92261fd4cec0b726692081c4d2966b75abf31dd"
|
||||||
_TENSORFLOW_SHA256 = "7f8a96dd99215c0cdc77230d3dbce43e60102b64a89203ad04aa09b0a187a4bd"
|
# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
|
||||||
|
_TENSORFLOW_SHA256 = "478a229bd4ec70a5b568ac23b5ea013d9fca46a47d6c43e30365a0412b9febf4"
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "org_tensorflow",
|
name = "org_tensorflow",
|
||||||
urls = [
|
urls = [
|
||||||
@@ -468,6 +512,7 @@ http_archive(
|
|||||||
],
|
],
|
||||||
patches = [
|
patches = [
|
||||||
"@//third_party:org_tensorflow_compatibility_fixes.diff",
|
"@//third_party:org_tensorflow_compatibility_fixes.diff",
|
||||||
|
"@//third_party:org_tensorflow_system_python.diff",
|
||||||
# Diff is generated with a script, don't update it manually.
|
# Diff is generated with a script, don't update it manually.
|
||||||
"@//third_party:org_tensorflow_custom_ops.diff",
|
"@//third_party:org_tensorflow_custom_ops.diff",
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
# Copyright 2022 The MediaPipe Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
@@ -14,6 +14,7 @@
|
|||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
"""Generate Java reference docs for MediaPipe."""
|
"""Generate Java reference docs for MediaPipe."""
|
||||||
import pathlib
|
import pathlib
|
||||||
|
import shutil
|
||||||
|
|
||||||
from absl import app
|
from absl import app
|
||||||
from absl import flags
|
from absl import flags
|
||||||
@@ -41,7 +42,9 @@ def main(_) -> None:
|
|||||||
mp_root = pathlib.Path(__file__)
|
mp_root = pathlib.Path(__file__)
|
||||||
while (mp_root := mp_root.parent).name != 'mediapipe':
|
while (mp_root := mp_root.parent).name != 'mediapipe':
|
||||||
# Find the nearest `mediapipe` dir.
|
# Find the nearest `mediapipe` dir.
|
||||||
pass
|
if not mp_root.name:
|
||||||
|
# We've hit the filesystem root - abort.
|
||||||
|
raise FileNotFoundError('"mediapipe" root not found')
|
||||||
|
|
||||||
# Find the root from which all packages are relative.
|
# Find the root from which all packages are relative.
|
||||||
root = mp_root.parent
|
root = mp_root.parent
|
||||||
@@ -51,6 +54,14 @@ def main(_) -> None:
|
|||||||
if (mp_root / 'mediapipe').exists():
|
if (mp_root / 'mediapipe').exists():
|
||||||
mp_root = mp_root / 'mediapipe'
|
mp_root = mp_root / 'mediapipe'
|
||||||
|
|
||||||
|
# We need to copy this into the tasks dir to ensure we don't leave broken
|
||||||
|
# links in the generated docs.
|
||||||
|
old_api_dir = 'java/com/google/mediapipe/framework/image'
|
||||||
|
shutil.copytree(
|
||||||
|
mp_root / old_api_dir,
|
||||||
|
mp_root / 'tasks' / old_api_dir,
|
||||||
|
dirs_exist_ok=True)
|
||||||
|
|
||||||
gen_java.gen_java_docs(
|
gen_java.gen_java_docs(
|
||||||
package='com.google.mediapipe',
|
package='com.google.mediapipe',
|
||||||
source_path=mp_root / 'tasks/java',
|
source_path=mp_root / 'tasks/java',
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
# Copyright 2022 The MediaPipe Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
# Copyright 2022 The MediaPipe Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
|||||||
3. The [`hello world`] example uses a simple MediaPipe graph in the
|
3. The [`hello world`] example uses a simple MediaPipe graph in the
|
||||||
`PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto.
|
`PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto.
|
||||||
|
|
||||||
```C++
|
```c++
|
||||||
absl::Status PrintHelloWorld() {
|
absl::Status PrintHelloWorld() {
|
||||||
// Configures a simple graph, which concatenates 2 PassThroughCalculators.
|
// Configures a simple graph, which concatenates 2 PassThroughCalculators.
|
||||||
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||||
@@ -126,7 +126,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
|||||||
```c++
|
```c++
|
||||||
mediapipe::Packet packet;
|
mediapipe::Packet packet;
|
||||||
while (poller.Next(&packet)) {
|
while (poller.Next(&packet)) {
|
||||||
LOG(INFO) << packet.Get<string>();
|
ABSL_LOG(INFO) << packet.Get<string>();
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -138,7 +138,7 @@ Create a `BUILD` file in the `$APPLICATION_PATH` and add the following build
|
|||||||
rules:
|
rules:
|
||||||
|
|
||||||
```
|
```
|
||||||
MIN_IOS_VERSION = "11.0"
|
MIN_IOS_VERSION = "12.0"
|
||||||
|
|
||||||
load(
|
load(
|
||||||
"@build_bazel_rules_apple//apple:ios.bzl",
|
"@build_bazel_rules_apple//apple:ios.bzl",
|
||||||
|
|||||||
+93
-107
@@ -1,99 +1,121 @@
|
|||||||
---
|
---
|
||||||
layout: default
|
layout: forward
|
||||||
|
target: https://developers.google.com/mediapipe
|
||||||
title: Home
|
title: Home
|
||||||
nav_order: 1
|
nav_order: 1
|
||||||
---
|
---
|
||||||
|
|
||||||

|
|
||||||
|
|
||||||
----
|
----
|
||||||
|
|
||||||
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
|
**Attention:** *We have moved to
|
||||||
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
|
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
|
||||||
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
||||||
|
|
||||||
*This notice and web page will be removed on June 1, 2023.*
|

|
||||||
|
|
||||||
----
|
**Attention**: MediaPipe Solutions Preview is an early release. [Learn
|
||||||
|
more](https://developers.google.com/mediapipe/solutions/about#notice).
|
||||||
|
|
||||||
<br><br><br><br><br><br><br><br><br><br>
|
**On-device machine learning for everyone**
|
||||||
<br><br><br><br><br><br><br><br><br><br>
|
|
||||||
<br><br><br><br><br><br><br><br><br><br>
|
|
||||||
|
|
||||||
--------------------------------------------------------------------------------
|
Delight your customers with innovative machine learning features. MediaPipe
|
||||||
|
contains everything that you need to customize and deploy to mobile (Android,
|
||||||
|
iOS), web, desktop, edge devices, and IoT, effortlessly.
|
||||||
|
|
||||||
## Live ML anywhere
|
* [See demos](https://goo.gle/mediapipe-studio)
|
||||||
|
* [Learn more](https://developers.google.com/mediapipe/solutions)
|
||||||
|
|
||||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
## Get started
|
||||||
ML solutions for live and streaming media.
|
|
||||||
|
|
||||||
 | 
|
You can get started with MediaPipe Solutions by by checking out any of the
|
||||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
developer guides for
|
||||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
|
||||||
 | 
|
[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
|
||||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
and
|
||||||
|
[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
|
||||||
|
tasks. If you need help setting up a development environment for use with
|
||||||
|
MediaPipe Tasks, check out the setup guides for
|
||||||
|
[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
|
||||||
|
apps](https://developers.google.com/mediapipe/solutions/setup_web), and
|
||||||
|
[Python](https://developers.google.com/mediapipe/solutions/setup_python).
|
||||||
|
|
||||||
----
|
## Solutions
|
||||||
|
|
||||||
## ML solutions in MediaPipe
|
MediaPipe Solutions provides a suite of libraries and tools for you to quickly
|
||||||
|
apply artificial intelligence (AI) and machine learning (ML) techniques in your
|
||||||
|
applications. You can plug these solutions into your applications immediately,
|
||||||
|
customize them to your needs, and use them across multiple development
|
||||||
|
platforms. MediaPipe Solutions is part of the MediaPipe [open source
|
||||||
|
project](https://github.com/google/mediapipe), so you can further customize the
|
||||||
|
solutions code to meet your application needs.
|
||||||
|
|
||||||
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
|
These libraries and resources provide the core functionality for each MediaPipe
|
||||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
|
Solution:
|
||||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
|
|
||||||
|
|
||||||
Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
|
||||||
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
solutions. [Learn
|
||||||
[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
more](https://developers.google.com/mediapipe/solutions/tasks).
|
||||||
|
* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
|
||||||
|
solution.
|
||||||
|
|
||||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
These tools let you customize and evaluate solutions:
|
||||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
|
||||||
|
|
||||||
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
* **MediaPipe Model Maker**: Customize models for solutions with your data.
|
||||||
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
|
[Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
|
||||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
|
* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
|
||||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
browser. [Learn
|
||||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
|
more](https://developers.google.com/mediapipe/solutions/studio).
|
||||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
||||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
||||||
[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
||||||
[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
||||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
|
|
||||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
|
||||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
|
||||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
|
||||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
|
|
||||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
|
||||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
|
||||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
|
||||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
|
||||||
|
|
||||||
See also
|
### Legacy solutions
|
||||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
|
||||||
for ML models released in MediaPipe.
|
|
||||||
|
|
||||||
## Getting started
|
We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||||
|
as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
|
||||||
|
a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||||
|
for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
|
||||||
|
and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
|
||||||
|
provided on an as-is basis.
|
||||||
|
|
||||||
To start using MediaPipe
|
For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
|
||||||
[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
|
|
||||||
lines code, see example code and demos in
|
|
||||||
[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
|
|
||||||
[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
|
|
||||||
|
|
||||||
To use MediaPipe in C++, Android and iOS, which allow further customization of
|
## Framework
|
||||||
the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
|
|
||||||
building your own, learn how to
|
|
||||||
[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
|
|
||||||
start building example applications in
|
|
||||||
[C++](https://google.github.io/mediapipe/getting_started/cpp),
|
|
||||||
[Android](https://google.github.io/mediapipe/getting_started/android) and
|
|
||||||
[iOS](https://google.github.io/mediapipe/getting_started/ios).
|
|
||||||
|
|
||||||
The source code is hosted in the
|
To start using MediaPipe Framework, [install MediaPipe
|
||||||
[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
|
Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
|
||||||
run code search using
|
and start building example applications in C++, Android, and iOS.
|
||||||
[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
|
|
||||||
|
|
||||||
## Publications
|
[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
|
||||||
|
low-level component used to build efficient on-device machine learning
|
||||||
|
pipelines, similar to the premade MediaPipe Solutions.
|
||||||
|
|
||||||
|
Before using MediaPipe Framework, familiarize yourself with the following key
|
||||||
|
[Framework
|
||||||
|
concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
|
||||||
|
|
||||||
|
* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
|
||||||
|
* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
|
||||||
|
* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
|
||||||
|
|
||||||
|
## Community
|
||||||
|
|
||||||
|
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
|
||||||
|
users.
|
||||||
|
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||||
|
community discussion around MediaPipe.
|
||||||
|
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||||
|
curated list of awesome MediaPipe related frameworks, libraries and
|
||||||
|
software.
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
We welcome contributions. Please follow these
|
||||||
|
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||||
|
|
||||||
|
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||||
|
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||||
|
|
||||||
|
## Resources
|
||||||
|
|
||||||
|
### Publications
|
||||||
|
|
||||||
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
||||||
in Google Developers Blog
|
in Google Developers Blog
|
||||||
@@ -102,7 +124,8 @@ run code search using
|
|||||||
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
||||||
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
||||||
in Google Developers Blog
|
in Google Developers Blog
|
||||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
|
||||||
|
Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||||
in Google AI Blog
|
in Google AI Blog
|
||||||
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
||||||
in Google AI Blog
|
in Google AI Blog
|
||||||
@@ -130,43 +153,6 @@ run code search using
|
|||||||
in Google AI Blog
|
in Google AI Blog
|
||||||
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
||||||
|
|
||||||
## Videos
|
### Videos
|
||||||
|
|
||||||
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
|
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
|
||||||
|
|
||||||
## Events
|
|
||||||
|
|
||||||
* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
|
|
||||||
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
|
|
||||||
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
|
|
||||||
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
|
|
||||||
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
|
|
||||||
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
|
|
||||||
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
|
|
||||||
Seoul, Korea ICCV
|
|
||||||
2019](https://research.google.com/youtube8m/workshop2019/index.html)
|
|
||||||
* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
|
|
||||||
* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
|
|
||||||
([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
|
|
||||||
* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
|
|
||||||
|
|
||||||
## Community
|
|
||||||
|
|
||||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
|
||||||
curated list of awesome MediaPipe related frameworks, libraries and software
|
|
||||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
|
|
||||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
|
||||||
community discussion around MediaPipe
|
|
||||||
|
|
||||||
## Alpha disclaimer
|
|
||||||
|
|
||||||
MediaPipe is currently in alpha at v0.7. We may be still making breaking API
|
|
||||||
changes and expect to get to stable APIs by v1.0.
|
|
||||||
|
|
||||||
## Contributing
|
|
||||||
|
|
||||||
We welcome contributions. Please follow these
|
|
||||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
|
||||||
|
|
||||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
|
||||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
|
||||||
|
|||||||
@@ -20,9 +20,9 @@ nav_order: 1
|
|||||||
---
|
---
|
||||||
|
|
||||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||||
Solution. For more information, see the
|
Solution. For more information, see the
|
||||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_detector)
|
||||||
site.*
|
site.*
|
||||||
|
|
||||||
----
|
----
|
||||||
|
|||||||
@@ -20,9 +20,9 @@ nav_order: 2
|
|||||||
---
|
---
|
||||||
|
|
||||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||||
Solution. For more information, see the
|
Solution. For more information, see the
|
||||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
|
||||||
site.*
|
site.*
|
||||||
|
|
||||||
----
|
----
|
||||||
|
|||||||
@@ -20,9 +20,9 @@ nav_order: 3
|
|||||||
---
|
---
|
||||||
|
|
||||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||||
Solution. For more information, see the
|
Solution. For more information, see the
|
||||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
|
||||||
site.*
|
site.*
|
||||||
|
|
||||||
----
|
----
|
||||||
|
|||||||
@@ -22,9 +22,9 @@ nav_order: 5
|
|||||||
---
|
---
|
||||||
|
|
||||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||||
Solution. For more information, see the
|
Solution. For more information, see the
|
||||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
|
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker)
|
||||||
site.*
|
site.*
|
||||||
|
|
||||||
----
|
----
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ nav_order: 1
|
|||||||
---
|
---
|
||||||
|
|
||||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||||
Solution. For more information, see the
|
Solution. For more information, see the
|
||||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
|
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
|
||||||
site.*
|
site.*
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
---
|
---
|
||||||
layout: default
|
layout: forward
|
||||||
|
target: https://developers.google.com/mediapipe/solutions/guide#legacy
|
||||||
title: MediaPipe Legacy Solutions
|
title: MediaPipe Legacy Solutions
|
||||||
nav_order: 3
|
nav_order: 3
|
||||||
has_children: true
|
has_children: true
|
||||||
@@ -13,8 +14,7 @@ has_toc: false
|
|||||||
{:toc}
|
{:toc}
|
||||||
---
|
---
|
||||||
|
|
||||||
**Attention:** *Thank you for your interest in MediaPipe Solutions. We have
|
**Attention:** *We have ended support for
|
||||||
ended support for
|
|
||||||
[these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
[these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||||
as of March 1, 2023. All other
|
as of March 1, 2023. All other
|
||||||
[MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
[MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||||
@@ -25,14 +25,6 @@ be provided on an as-is basis. We encourage you to check out the new MediaPipe
|
|||||||
Solutions at:
|
Solutions at:
|
||||||
[https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
|
[https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
|
||||||
|
|
||||||
*This notice and web page will be removed on June 1, 2023.*
|
|
||||||
|
|
||||||
----
|
|
||||||
|
|
||||||
<br><br><br><br><br><br><br><br><br><br>
|
|
||||||
<br><br><br><br><br><br><br><br><br><br>
|
|
||||||
<br><br><br><br><br><br><br><br><br><br>
|
|
||||||
|
|
||||||
----
|
----
|
||||||
|
|
||||||
MediaPipe offers open source cross-platform, customizable ML solutions for live
|
MediaPipe offers open source cross-platform, customizable ML solutions for live
|
||||||
|
|||||||
+138
-91
@@ -14,81 +14,155 @@
|
|||||||
|
|
||||||
licenses(["notice"]) # Apache 2.0
|
licenses(["notice"]) # Apache 2.0
|
||||||
|
|
||||||
# Note: yes, these need to use "//external:android/crosstool", not
|
load("@mediapipe//mediapipe:platforms.bzl", "config_setting_and_platform")
|
||||||
# @androidndk//:default_crosstool.
|
|
||||||
|
|
||||||
|
# Generic Android
|
||||||
config_setting(
|
config_setting(
|
||||||
name = "android",
|
name = "android",
|
||||||
values = {"crosstool_top": "//external:android/crosstool"},
|
constraint_values = [
|
||||||
|
"@platforms//os:android",
|
||||||
|
],
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
config_setting(
|
# Android x86 32-bit.
|
||||||
|
config_setting_and_platform(
|
||||||
name = "android_x86",
|
name = "android_x86",
|
||||||
values = {
|
constraint_values = [
|
||||||
"crosstool_top": "//external:android/crosstool",
|
"@platforms//os:android",
|
||||||
"cpu": "x86",
|
"@platforms//cpu:x86_32",
|
||||||
},
|
],
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
config_setting(
|
# Android x86 64-bit.
|
||||||
|
config_setting_and_platform(
|
||||||
name = "android_x86_64",
|
name = "android_x86_64",
|
||||||
values = {
|
constraint_values = [
|
||||||
"crosstool_top": "//external:android/crosstool",
|
"@platforms//os:android",
|
||||||
"cpu": "x86_64",
|
"@platforms//cpu:x86_64",
|
||||||
},
|
],
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
config_setting(
|
# Android ARMv7.
|
||||||
name = "android_armeabi",
|
config_setting_and_platform(
|
||||||
values = {
|
|
||||||
"crosstool_top": "//external:android/crosstool",
|
|
||||||
"cpu": "armeabi",
|
|
||||||
},
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
config_setting(
|
|
||||||
name = "android_arm",
|
name = "android_arm",
|
||||||
values = {
|
constraint_values = [
|
||||||
"crosstool_top": "//external:android/crosstool",
|
"@platforms//os:android",
|
||||||
"cpu": "armeabi-v7a",
|
"@platforms//cpu:armv7",
|
||||||
},
|
],
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
config_setting(
|
# Android ARM64.
|
||||||
|
config_setting_and_platform(
|
||||||
name = "android_arm64",
|
name = "android_arm64",
|
||||||
values = {
|
constraint_values = [
|
||||||
"crosstool_top": "//external:android/crosstool",
|
"@platforms//os:android",
|
||||||
"cpu": "arm64-v8a",
|
"@platforms//cpu:arm64",
|
||||||
},
|
],
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Note: this cannot just match "apple_platform_type": "macos" because that option
|
# Generic MacOS.
|
||||||
# defaults to "macos" even when building on Linux!
|
config_setting(
|
||||||
alias(
|
|
||||||
name = "macos",
|
name = "macos",
|
||||||
actual = select({
|
constraint_values = [
|
||||||
":macos_i386": ":macos_i386",
|
"@platforms//os:macos",
|
||||||
":macos_x86_64": ":macos_x86_64",
|
],
|
||||||
":macos_arm64": ":macos_arm64",
|
|
||||||
"//conditions:default": ":macos_i386", # Arbitrarily chosen from above.
|
|
||||||
}),
|
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Note: this also matches on crosstool_top so that it does not produce ambiguous
|
# MacOS x86 64-bit.
|
||||||
# selectors when used together with "android".
|
config_setting_and_platform(
|
||||||
|
name = "macos_x86_64",
|
||||||
|
constraint_values = [
|
||||||
|
"@platforms//os:macos",
|
||||||
|
"@platforms//cpu:x86_64",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# MacOS ARM64.
|
||||||
|
config_setting_and_platform(
|
||||||
|
name = "macos_arm64",
|
||||||
|
constraint_values = [
|
||||||
|
"@platforms//os:macos",
|
||||||
|
"@platforms//cpu:arm64",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Generic iOS.
|
||||||
config_setting(
|
config_setting(
|
||||||
name = "ios",
|
name = "ios",
|
||||||
values = {
|
constraint_values = [
|
||||||
"crosstool_top": "@bazel_tools//tools/cpp:toolchain",
|
"@platforms//os:ios",
|
||||||
"apple_platform_type": "ios",
|
],
|
||||||
},
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# iOS device ARM32.
|
||||||
|
config_setting_and_platform(
|
||||||
|
name = "ios_armv7",
|
||||||
|
constraint_values = [
|
||||||
|
"@platforms//os:ios",
|
||||||
|
"@platforms//cpu:arm",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# iOS device ARM64.
|
||||||
|
config_setting_and_platform(
|
||||||
|
name = "ios_arm64",
|
||||||
|
constraint_values = [
|
||||||
|
"@platforms//os:ios",
|
||||||
|
"@platforms//cpu:arm64",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# iOS device ARM64E.
|
||||||
|
config_setting_and_platform(
|
||||||
|
name = "ios_arm64e",
|
||||||
|
constraint_values = [
|
||||||
|
"@platforms//os:ios",
|
||||||
|
"@platforms//cpu:arm64e",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# iOS simulator x86 32-bit.
|
||||||
|
config_setting_and_platform(
|
||||||
|
name = "ios_i386",
|
||||||
|
constraint_values = [
|
||||||
|
"@platforms//os:ios",
|
||||||
|
"@platforms//cpu:x86_32",
|
||||||
|
"@build_bazel_apple_support//constraints:simulator",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# iOS simulator x86 64-bit.
|
||||||
|
config_setting_and_platform(
|
||||||
|
name = "ios_x86_64",
|
||||||
|
constraint_values = [
|
||||||
|
"@platforms//os:ios",
|
||||||
|
"@platforms//cpu:x86_64",
|
||||||
|
"@build_bazel_apple_support//constraints:simulator",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# iOS simulator ARM64.
|
||||||
|
config_setting_and_platform(
|
||||||
|
name = "ios_sim_arm64",
|
||||||
|
constraint_values = [
|
||||||
|
"@platforms//os:ios",
|
||||||
|
"@platforms//cpu:arm64",
|
||||||
|
"@build_bazel_apple_support//constraints:simulator",
|
||||||
|
],
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -102,51 +176,24 @@ alias(
|
|||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
config_setting(
|
# Windows 64-bit.
|
||||||
name = "macos_i386",
|
config_setting_and_platform(
|
||||||
values = {
|
|
||||||
"apple_platform_type": "macos",
|
|
||||||
"cpu": "darwin",
|
|
||||||
},
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
config_setting(
|
|
||||||
name = "macos_x86_64",
|
|
||||||
values = {
|
|
||||||
"apple_platform_type": "macos",
|
|
||||||
"cpu": "darwin_x86_64",
|
|
||||||
},
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
config_setting(
|
|
||||||
name = "macos_arm64",
|
|
||||||
values = {
|
|
||||||
"apple_platform_type": "macos",
|
|
||||||
"cpu": "darwin_arm64",
|
|
||||||
},
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
[
|
|
||||||
config_setting(
|
|
||||||
name = arch,
|
|
||||||
values = {"cpu": arch},
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
for arch in [
|
|
||||||
"ios_i386",
|
|
||||||
"ios_x86_64",
|
|
||||||
"ios_armv7",
|
|
||||||
"ios_arm64",
|
|
||||||
"ios_arm64e",
|
|
||||||
]
|
|
||||||
]
|
|
||||||
|
|
||||||
config_setting(
|
|
||||||
name = "windows",
|
name = "windows",
|
||||||
values = {"cpu": "x64_windows"},
|
constraint_values = [
|
||||||
|
"@platforms//os:windows",
|
||||||
|
"@platforms//cpu:x86_64",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Linux 64-bit.
|
||||||
|
config_setting_and_platform(
|
||||||
|
name = "linux",
|
||||||
|
constraint_values = [
|
||||||
|
"@platforms//os:linux",
|
||||||
|
"@platforms//cpu:x86_64",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
exports_files(
|
exports_files(
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
# Placeholder: load py_proto_library
|
||||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||||
|
|
||||||
licenses(["notice"])
|
licenses(["notice"])
|
||||||
@@ -145,6 +146,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:logging",
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:time_series_util",
|
"//mediapipe/util:time_series_util",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@com_google_audio_tools//audio/dsp/mfcc",
|
"@com_google_audio_tools//audio/dsp/mfcc",
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
@@ -163,8 +165,9 @@ cc_library(
|
|||||||
"//mediapipe/framework/formats:matrix",
|
"//mediapipe/framework/formats:matrix",
|
||||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
"//mediapipe/framework/port:logging",
|
|
||||||
"//mediapipe/util:time_series_util",
|
"//mediapipe/util:time_series_util",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@com_google_audio_tools//audio/dsp:resampler",
|
"@com_google_audio_tools//audio/dsp:resampler",
|
||||||
"@com_google_audio_tools//audio/dsp:resampler_q",
|
"@com_google_audio_tools//audio/dsp:resampler_q",
|
||||||
@@ -185,6 +188,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:core_proto",
|
"//mediapipe/framework/port:core_proto",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:time_series_util",
|
"//mediapipe/util:time_series_util",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
@@ -219,13 +223,12 @@ cc_library(
|
|||||||
deps = [
|
deps = [
|
||||||
":time_series_framer_calculator_cc_proto",
|
":time_series_framer_calculator_cc_proto",
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
|
"//mediapipe/framework:timestamp",
|
||||||
"//mediapipe/framework/formats:matrix",
|
"//mediapipe/framework/formats:matrix",
|
||||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||||
"//mediapipe/framework/port:integral_types",
|
|
||||||
"//mediapipe/framework/port:logging",
|
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
|
||||||
"//mediapipe/util:time_series_util",
|
"//mediapipe/util:time_series_util",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
],
|
],
|
||||||
@@ -296,6 +299,7 @@ cc_test(
|
|||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:time_series_test_util",
|
"//mediapipe/util:time_series_test_util",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
"@com_google_audio_tools//audio/dsp:number_util",
|
"@com_google_audio_tools//audio/dsp:number_util",
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
],
|
],
|
||||||
@@ -319,6 +323,21 @@ cc_test(
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cc_binary(
|
||||||
|
name = "time_series_framer_calculator_benchmark",
|
||||||
|
srcs = ["time_series_framer_calculator_benchmark.cc"],
|
||||||
|
deps = [
|
||||||
|
":time_series_framer_calculator",
|
||||||
|
":time_series_framer_calculator_cc_proto",
|
||||||
|
"//mediapipe/framework:calculator_framework",
|
||||||
|
"//mediapipe/framework:packet",
|
||||||
|
"//mediapipe/framework/formats:matrix",
|
||||||
|
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
|
"@com_google_benchmark//:benchmark",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
cc_test(
|
cc_test(
|
||||||
name = "time_series_framer_calculator_test",
|
name = "time_series_framer_calculator_test",
|
||||||
srcs = ["time_series_framer_calculator_test.cc"],
|
srcs = ["time_series_framer_calculator_test.cc"],
|
||||||
@@ -333,6 +352,7 @@ cc_test(
|
|||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:time_series_test_util",
|
"//mediapipe/util:time_series_test_util",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -23,6 +23,7 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "absl/strings/str_cat.h"
|
#include "absl/strings/str_cat.h"
|
||||||
#include "absl/strings/string_view.h"
|
#include "absl/strings/string_view.h"
|
||||||
#include "absl/strings/substitute.h"
|
#include "absl/strings/substitute.h"
|
||||||
@@ -138,7 +139,7 @@ absl::Status FramewiseTransformCalculatorBase::Process(CalculatorContext* cc) {
|
|||||||
TransformFrame(input_frame, &output_frame);
|
TransformFrame(input_frame, &output_frame);
|
||||||
|
|
||||||
// Copy output from vector<float> to Eigen::Vector.
|
// Copy output from vector<float> to Eigen::Vector.
|
||||||
CHECK_EQ(output_frame.size(), num_output_channels_);
|
ABSL_CHECK_EQ(output_frame.size(), num_output_channels_);
|
||||||
Eigen::Map<const Eigen::MatrixXd> output_frame_map(&output_frame[0],
|
Eigen::Map<const Eigen::MatrixXd> output_frame_map(&output_frame[0],
|
||||||
output_frame.size(), 1);
|
output_frame.size(), 1);
|
||||||
output->col(frame) = output_frame_map.cast<float>();
|
output->col(frame) = output_frame_map.cast<float>();
|
||||||
|
|||||||
@@ -16,6 +16,8 @@
|
|||||||
|
|
||||||
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
|
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "audio/dsp/resampler_q.h"
|
#include "audio/dsp/resampler_q.h"
|
||||||
|
|
||||||
using audio_dsp::Resampler;
|
using audio_dsp::Resampler;
|
||||||
@@ -45,9 +47,9 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
|
|||||||
if (matrix->cols() == 0) {
|
if (matrix->cols() == 0) {
|
||||||
matrix->resize(matrix->rows(), vec.size());
|
matrix->resize(matrix->rows(), vec.size());
|
||||||
} else {
|
} else {
|
||||||
CHECK_EQ(vec.size(), matrix->cols());
|
ABSL_CHECK_EQ(vec.size(), matrix->cols());
|
||||||
}
|
}
|
||||||
CHECK_LT(channel, matrix->rows());
|
ABSL_CHECK_LT(channel, matrix->rows());
|
||||||
matrix->row(channel) =
|
matrix->row(channel) =
|
||||||
Eigen::Map<const Eigen::ArrayXf>(vec.data(), vec.size());
|
Eigen::Map<const Eigen::ArrayXf>(vec.data(), vec.size());
|
||||||
}
|
}
|
||||||
@@ -77,7 +79,7 @@ absl::Status RationalFactorResampleCalculator::Open(CalculatorContext* cc) {
|
|||||||
r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_,
|
r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_,
|
||||||
resample_options);
|
resample_options);
|
||||||
if (!r) {
|
if (!r) {
|
||||||
LOG(ERROR) << "Failed to initialize resampler.";
|
ABSL_LOG(ERROR) << "Failed to initialize resampler.";
|
||||||
return absl::UnknownError("Failed to initialize resampler.");
|
return absl::UnknownError("Failed to initialize resampler.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,7 +27,6 @@
|
|||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||||
#include "mediapipe/framework/port/integral_types.h"
|
#include "mediapipe/framework/port/integral_types.h"
|
||||||
#include "mediapipe/framework/port/logging.h"
|
|
||||||
#include "mediapipe/util/time_series_util.h"
|
#include "mediapipe/util/time_series_util.h"
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
|
|||||||
@@ -210,6 +210,23 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
|
|||||||
// Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
|
// Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
|
||||||
const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;
|
const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
std::unique_ptr<audio_dsp::WindowFunction> MakeWindowFun(
|
||||||
|
const SpectrogramCalculatorOptions::WindowType window_type) {
|
||||||
|
switch (window_type) {
|
||||||
|
// The cosine window and square root of Hann are equivalent.
|
||||||
|
case SpectrogramCalculatorOptions::COSINE:
|
||||||
|
case SpectrogramCalculatorOptions::SQRT_HANN:
|
||||||
|
return std::make_unique<audio_dsp::CosineWindow>();
|
||||||
|
case SpectrogramCalculatorOptions::HANN:
|
||||||
|
return std::make_unique<audio_dsp::HannWindow>();
|
||||||
|
case SpectrogramCalculatorOptions::HAMMING:
|
||||||
|
return std::make_unique<audio_dsp::HammingWindow>();
|
||||||
|
}
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
|
absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
|
||||||
SpectrogramCalculatorOptions spectrogram_options =
|
SpectrogramCalculatorOptions spectrogram_options =
|
||||||
cc->Options<SpectrogramCalculatorOptions>();
|
cc->Options<SpectrogramCalculatorOptions>();
|
||||||
@@ -266,28 +283,14 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
|
|||||||
|
|
||||||
output_scale_ = spectrogram_options.output_scale();
|
output_scale_ = spectrogram_options.output_scale();
|
||||||
|
|
||||||
std::vector<double> window;
|
auto window_fun = MakeWindowFun(spectrogram_options.window_type());
|
||||||
switch (spectrogram_options.window_type()) {
|
if (window_fun == nullptr) {
|
||||||
case SpectrogramCalculatorOptions::COSINE:
|
return absl::Status(absl::StatusCode::kInvalidArgument,
|
||||||
audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
|
absl::StrCat("Invalid window type ",
|
||||||
&window);
|
spectrogram_options.window_type()));
|
||||||
break;
|
|
||||||
case SpectrogramCalculatorOptions::HANN:
|
|
||||||
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
|
|
||||||
&window);
|
|
||||||
break;
|
|
||||||
case SpectrogramCalculatorOptions::HAMMING:
|
|
||||||
audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
|
|
||||||
&window);
|
|
||||||
break;
|
|
||||||
case SpectrogramCalculatorOptions::SQRT_HANN: {
|
|
||||||
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
|
|
||||||
&window);
|
|
||||||
absl::c_transform(window, window.begin(),
|
|
||||||
[](double x) { return std::sqrt(x); });
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
std::vector<double> window;
|
||||||
|
window_fun->GetPeriodicSamples(frame_duration_samples_, &window);
|
||||||
|
|
||||||
// Propagate settings down to the actual Spectrogram object.
|
// Propagate settings down to the actual Spectrogram object.
|
||||||
spectrogram_generators_.clear();
|
spectrogram_generators_.clear();
|
||||||
@@ -433,9 +436,9 @@ absl::Status SpectrogramCalculator::ProcessVectorToOutput(
|
|||||||
absl::Status SpectrogramCalculator::ProcessVector(const Matrix& input_stream,
|
absl::Status SpectrogramCalculator::ProcessVector(const Matrix& input_stream,
|
||||||
CalculatorContext* cc) {
|
CalculatorContext* cc) {
|
||||||
switch (output_type_) {
|
switch (output_type_) {
|
||||||
// These blocks deliberately ignore clang-format to preserve the
|
// These blocks deliberately ignore clang-format to preserve the
|
||||||
// "silhouette" of the different cases.
|
// "silhouette" of the different cases.
|
||||||
// clang-format off
|
// clang-format off
|
||||||
case SpectrogramCalculatorOptions::COMPLEX: {
|
case SpectrogramCalculatorOptions::COMPLEX: {
|
||||||
return ProcessVectorToOutput(
|
return ProcessVectorToOutput(
|
||||||
input_stream,
|
input_stream,
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ message SpectrogramCalculatorOptions {
|
|||||||
HANN = 0;
|
HANN = 0;
|
||||||
HAMMING = 1;
|
HAMMING = 1;
|
||||||
COSINE = 2;
|
COSINE = 2;
|
||||||
SQRT_HANN = 4;
|
SQRT_HANN = 4; // Alias of COSINE.
|
||||||
}
|
}
|
||||||
optional WindowType window_type = 6 [default = HANN];
|
optional WindowType window_type = 6 [default = HANN];
|
||||||
|
|
||||||
|
|||||||
@@ -22,6 +22,7 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "audio/dsp/number_util.h"
|
#include "audio/dsp/number_util.h"
|
||||||
#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
|
#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
@@ -882,11 +883,11 @@ void BM_ProcessDC(benchmark::State& state) {
|
|||||||
|
|
||||||
const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0);
|
const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0);
|
||||||
const Matrix& output_matrix = output.packets[0].Get<Matrix>();
|
const Matrix& output_matrix = output.packets[0].Get<Matrix>();
|
||||||
LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
|
ABSL_LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
|
||||||
<< output_matrix.cols();
|
<< output_matrix.cols();
|
||||||
LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
|
ABSL_LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
|
||||||
<< output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
|
<< output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
|
||||||
<< output_matrix(3, 0);
|
<< output_matrix(3, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
BENCHMARK(BM_ProcessDC);
|
BENCHMARK(BM_ProcessDC);
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
|
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
@@ -59,7 +60,7 @@ class StabilizedLogCalculator : public CalculatorBase {
|
|||||||
output_scale_ = stabilized_log_calculator_options.output_scale();
|
output_scale_ = stabilized_log_calculator_options.output_scale();
|
||||||
check_nonnegativity_ =
|
check_nonnegativity_ =
|
||||||
stabilized_log_calculator_options.check_nonnegativity();
|
stabilized_log_calculator_options.check_nonnegativity();
|
||||||
CHECK_GE(stabilizer_, 0.0)
|
ABSL_CHECK_GE(stabilizer_, 0.0)
|
||||||
<< "stabilizer must be >= 0.0, received a value of " << stabilizer_;
|
<< "stabilizer must be >= 0.0, received a value of " << stabilizer_;
|
||||||
|
|
||||||
// If the input packets have a header, propagate the header to the output.
|
// If the input packets have a header, propagate the header to the output.
|
||||||
|
|||||||
@@ -15,19 +15,17 @@
|
|||||||
// Defines TimeSeriesFramerCalculator.
|
// Defines TimeSeriesFramerCalculator.
|
||||||
#include <math.h>
|
#include <math.h>
|
||||||
|
|
||||||
#include <deque>
|
#include <vector>
|
||||||
#include <memory>
|
|
||||||
#include <string>
|
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "audio/dsp/window_functions.h"
|
#include "audio/dsp/window_functions.h"
|
||||||
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||||
#include "mediapipe/framework/port/integral_types.h"
|
|
||||||
#include "mediapipe/framework/port/logging.h"
|
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
|
#include "mediapipe/framework/timestamp.h"
|
||||||
#include "mediapipe/util/time_series_util.h"
|
#include "mediapipe/util/time_series_util.h"
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
@@ -88,11 +86,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
|
|||||||
absl::Status Close(CalculatorContext* cc) override;
|
absl::Status Close(CalculatorContext* cc) override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
// Adds input data to the internal buffer.
|
|
||||||
void EnqueueInput(CalculatorContext* cc);
|
|
||||||
// Constructs and emits framed output packets.
|
|
||||||
void FrameOutput(CalculatorContext* cc);
|
|
||||||
|
|
||||||
Timestamp CurrentOutputTimestamp() {
|
Timestamp CurrentOutputTimestamp() {
|
||||||
if (use_local_timestamp_) {
|
if (use_local_timestamp_) {
|
||||||
return current_timestamp_;
|
return current_timestamp_;
|
||||||
@@ -106,21 +99,13 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
|
|||||||
Timestamp::kTimestampUnitsPerSecond);
|
Timestamp::kTimestampUnitsPerSecond);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns the timestamp of a sample on a base, which is usually the time
|
|
||||||
// stamp of a packet.
|
|
||||||
Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
|
|
||||||
int64_t number_of_samples) {
|
|
||||||
return timestamp_base + round(number_of_samples / sample_rate_ *
|
|
||||||
Timestamp::kTimestampUnitsPerSecond);
|
|
||||||
}
|
|
||||||
|
|
||||||
// The number of input samples to advance after the current output frame is
|
// The number of input samples to advance after the current output frame is
|
||||||
// emitted.
|
// emitted.
|
||||||
int next_frame_step_samples() const {
|
int next_frame_step_samples() const {
|
||||||
// All numbers are in input samples.
|
// All numbers are in input samples.
|
||||||
const int64_t current_output_frame_start = static_cast<int64_t>(
|
const int64_t current_output_frame_start = static_cast<int64_t>(
|
||||||
round(cumulative_output_frames_ * average_frame_step_samples_));
|
round(cumulative_output_frames_ * average_frame_step_samples_));
|
||||||
CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
|
ABSL_CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
|
||||||
const int64_t next_output_frame_start = static_cast<int64_t>(
|
const int64_t next_output_frame_start = static_cast<int64_t>(
|
||||||
round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
|
round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
|
||||||
return next_output_frame_start - current_output_frame_start;
|
return next_output_frame_start - current_output_frame_start;
|
||||||
@@ -142,61 +127,174 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
|
|||||||
Timestamp initial_input_timestamp_;
|
Timestamp initial_input_timestamp_;
|
||||||
// The current timestamp is updated along with the incoming packets.
|
// The current timestamp is updated along with the incoming packets.
|
||||||
Timestamp current_timestamp_;
|
Timestamp current_timestamp_;
|
||||||
int num_channels_;
|
|
||||||
|
|
||||||
// Each entry in this deque consists of a single sample, i.e. a
|
// Samples are buffered in a vector of sample blocks.
|
||||||
// single column vector, and its timestamp.
|
class SampleBlockBuffer {
|
||||||
std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
|
public:
|
||||||
|
// Initializes the buffer.
|
||||||
|
void Init(double sample_rate, int num_channels) {
|
||||||
|
ts_units_per_sample_ = Timestamp::kTimestampUnitsPerSecond / sample_rate;
|
||||||
|
num_channels_ = num_channels;
|
||||||
|
num_samples_ = 0;
|
||||||
|
first_block_offset_ = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Number of channels, equal to the number of rows in each Matrix.
|
||||||
|
int num_channels() const { return num_channels_; }
|
||||||
|
// Total number of available samples over all blocks.
|
||||||
|
int num_samples() const { return num_samples_; }
|
||||||
|
|
||||||
|
// Pushes a new block of samples on the back of the buffer with `timestamp`
|
||||||
|
// being the input timestamp of the packet containing the Matrix.
|
||||||
|
void Push(const Matrix& samples, Timestamp timestamp);
|
||||||
|
// Copies `count` samples from the front of the buffer. If there are fewer
|
||||||
|
// samples than this, the result is zero padded to have `count` samples.
|
||||||
|
// The timestamp of the last copied sample is written to *last_timestamp.
|
||||||
|
// This output is used below to update `current_timestamp_`, which is only
|
||||||
|
// used when `use_local_timestamp` is true.
|
||||||
|
Matrix CopySamples(int count, Timestamp* last_timestamp) const;
|
||||||
|
// Drops `count` samples from the front of the buffer. If `count` exceeds
|
||||||
|
// `num_samples()`, the buffer is emptied. Returns how many samples were
|
||||||
|
// dropped.
|
||||||
|
int DropSamples(int count);
|
||||||
|
|
||||||
|
private:
|
||||||
|
struct Block {
|
||||||
|
// Matrix of num_channels rows by num_samples columns, a block of possibly
|
||||||
|
// multiple samples.
|
||||||
|
Matrix samples;
|
||||||
|
// Timestamp of the first sample in the Block. This comes from the input
|
||||||
|
// packet's timestamp that contains this Matrix.
|
||||||
|
Timestamp timestamp;
|
||||||
|
|
||||||
|
Block() : timestamp(Timestamp::Unstarted()) {}
|
||||||
|
Block(const Matrix& samples, Timestamp timestamp)
|
||||||
|
: samples(samples), timestamp(timestamp) {}
|
||||||
|
int num_samples() const { return samples.cols(); }
|
||||||
|
};
|
||||||
|
std::vector<Block> blocks_;
|
||||||
|
// Number of timestamp units per sample. Used to compute timestamps as
|
||||||
|
// nth sample timestamp = base_timestamp + round(ts_units_per_sample_ * n).
|
||||||
|
double ts_units_per_sample_;
|
||||||
|
// Number of rows in each Matrix.
|
||||||
|
int num_channels_;
|
||||||
|
// The total number of samples over all blocks, equal to
|
||||||
|
// (sum_i blocks_[i].num_samples()) - first_block_offset_.
|
||||||
|
int num_samples_;
|
||||||
|
// The number of samples in the first block that have been discarded. This
|
||||||
|
// way we can cheaply represent "partially discarding" a block.
|
||||||
|
int first_block_offset_;
|
||||||
|
} sample_buffer_;
|
||||||
|
|
||||||
bool use_window_;
|
bool use_window_;
|
||||||
Matrix window_;
|
Eigen::RowVectorXf window_;
|
||||||
|
|
||||||
bool use_local_timestamp_;
|
bool use_local_timestamp_;
|
||||||
};
|
};
|
||||||
REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
|
REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
|
||||||
|
|
||||||
void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
|
void TimeSeriesFramerCalculator::SampleBlockBuffer::Push(const Matrix& samples,
|
||||||
const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
|
Timestamp timestamp) {
|
||||||
|
num_samples_ += samples.cols();
|
||||||
for (int i = 0; i < input_frame.cols(); ++i) {
|
blocks_.emplace_back(samples, timestamp);
|
||||||
sample_buffer_.emplace_back(std::make_pair(
|
|
||||||
input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
|
Matrix TimeSeriesFramerCalculator::SampleBlockBuffer::CopySamples(
|
||||||
while (sample_buffer_.size() >=
|
int count, Timestamp* last_timestamp) const {
|
||||||
|
Matrix copied(num_channels_, count);
|
||||||
|
|
||||||
|
if (!blocks_.empty()) {
|
||||||
|
int num_copied = 0;
|
||||||
|
// First block has an offset for samples that have been discarded.
|
||||||
|
int offset = first_block_offset_;
|
||||||
|
int n;
|
||||||
|
Timestamp last_block_ts;
|
||||||
|
int last_sample_index;
|
||||||
|
|
||||||
|
for (auto it = blocks_.begin(); it != blocks_.end() && count > 0; ++it) {
|
||||||
|
n = std::min(it->num_samples() - offset, count);
|
||||||
|
// Copy `n` samples from the next block.
|
||||||
|
copied.middleCols(num_copied, n) = it->samples.middleCols(offset, n);
|
||||||
|
count -= n;
|
||||||
|
num_copied += n;
|
||||||
|
last_block_ts = it->timestamp;
|
||||||
|
last_sample_index = offset + n - 1;
|
||||||
|
offset = 0; // No samples have been discarded in subsequent blocks.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute the timestamp of the last copied sample.
|
||||||
|
*last_timestamp =
|
||||||
|
last_block_ts + std::round(ts_units_per_sample_ * last_sample_index);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (count > 0) {
|
||||||
|
copied.rightCols(count).setZero(); // Zero pad if needed.
|
||||||
|
}
|
||||||
|
|
||||||
|
return copied;
|
||||||
|
}
|
||||||
|
|
||||||
|
int TimeSeriesFramerCalculator::SampleBlockBuffer::DropSamples(int count) {
|
||||||
|
if (blocks_.empty()) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto block_it = blocks_.begin();
|
||||||
|
if (first_block_offset_ + count < block_it->num_samples()) {
|
||||||
|
// `count` is less than the remaining samples in the first block.
|
||||||
|
first_block_offset_ += count;
|
||||||
|
num_samples_ -= count;
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
int num_samples_dropped = block_it->num_samples() - first_block_offset_;
|
||||||
|
count -= num_samples_dropped;
|
||||||
|
first_block_offset_ = 0;
|
||||||
|
|
||||||
|
for (++block_it; block_it != blocks_.end(); ++block_it) {
|
||||||
|
if (block_it->num_samples() > count) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
num_samples_dropped += block_it->num_samples();
|
||||||
|
count -= block_it->num_samples();
|
||||||
|
}
|
||||||
|
|
||||||
|
blocks_.erase(blocks_.begin(), block_it); // Drop whole blocks.
|
||||||
|
if (!blocks_.empty()) {
|
||||||
|
first_block_offset_ = count; // Drop part of the next block.
|
||||||
|
num_samples_dropped += count;
|
||||||
|
}
|
||||||
|
|
||||||
|
num_samples_ -= num_samples_dropped;
|
||||||
|
return num_samples_dropped;
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
|
||||||
|
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
|
||||||
|
initial_input_timestamp_ = cc->InputTimestamp();
|
||||||
|
current_timestamp_ = initial_input_timestamp_;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add input data to the internal buffer.
|
||||||
|
sample_buffer_.Push(cc->Inputs().Index(0).Get<Matrix>(),
|
||||||
|
cc->InputTimestamp());
|
||||||
|
|
||||||
|
// Construct and emit framed output packets.
|
||||||
|
while (sample_buffer_.num_samples() >=
|
||||||
frame_duration_samples_ + samples_still_to_drop_) {
|
frame_duration_samples_ + samples_still_to_drop_) {
|
||||||
while (samples_still_to_drop_ > 0) {
|
sample_buffer_.DropSamples(samples_still_to_drop_);
|
||||||
sample_buffer_.pop_front();
|
Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
|
||||||
--samples_still_to_drop_;
|
¤t_timestamp_);
|
||||||
}
|
|
||||||
const int frame_step_samples = next_frame_step_samples();
|
const int frame_step_samples = next_frame_step_samples();
|
||||||
std::unique_ptr<Matrix> output_frame(
|
samples_still_to_drop_ = frame_step_samples;
|
||||||
new Matrix(num_channels_, frame_duration_samples_));
|
|
||||||
for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
|
|
||||||
++i) {
|
|
||||||
output_frame->col(i) = sample_buffer_.front().first;
|
|
||||||
current_timestamp_ = sample_buffer_.front().second;
|
|
||||||
sample_buffer_.pop_front();
|
|
||||||
}
|
|
||||||
const int frame_overlap_samples =
|
|
||||||
frame_duration_samples_ - frame_step_samples;
|
|
||||||
if (frame_overlap_samples > 0) {
|
|
||||||
for (int i = 0; i < frame_overlap_samples; ++i) {
|
|
||||||
output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
|
|
||||||
current_timestamp_ = sample_buffer_[i].second;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
samples_still_to_drop_ = -frame_overlap_samples;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (use_window_) {
|
if (use_window_) {
|
||||||
*output_frame = (output_frame->array() * window_.array()).matrix();
|
// Apply the window to each row of output_frame.
|
||||||
|
output_frame.array().rowwise() *= window_.array();
|
||||||
}
|
}
|
||||||
|
|
||||||
cc->Outputs().Index(0).Add(output_frame.release(),
|
cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
|
||||||
CurrentOutputTimestamp());
|
.At(CurrentOutputTimestamp()));
|
||||||
++cumulative_output_frames_;
|
++cumulative_output_frames_;
|
||||||
cumulative_completed_samples_ += frame_step_samples;
|
cumulative_completed_samples_ += frame_step_samples;
|
||||||
}
|
}
|
||||||
@@ -206,35 +304,18 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
|
|||||||
// fact to enable packet queueing optimizations.
|
// fact to enable packet queueing optimizations.
|
||||||
cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
|
cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
|
|
||||||
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
|
|
||||||
initial_input_timestamp_ = cc->InputTimestamp();
|
|
||||||
current_timestamp_ = initial_input_timestamp_;
|
|
||||||
}
|
|
||||||
|
|
||||||
EnqueueInput(cc);
|
|
||||||
FrameOutput(cc);
|
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
|
absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
|
||||||
while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
|
sample_buffer_.DropSamples(samples_still_to_drop_);
|
||||||
sample_buffer_.pop_front();
|
|
||||||
--samples_still_to_drop_;
|
|
||||||
}
|
|
||||||
if (!sample_buffer_.empty() && pad_final_packet_) {
|
|
||||||
std::unique_ptr<Matrix> output_frame(new Matrix);
|
|
||||||
output_frame->setZero(num_channels_, frame_duration_samples_);
|
|
||||||
for (int i = 0; i < sample_buffer_.size(); ++i) {
|
|
||||||
output_frame->col(i) = sample_buffer_[i].first;
|
|
||||||
current_timestamp_ = sample_buffer_[i].second;
|
|
||||||
}
|
|
||||||
|
|
||||||
cc->Outputs().Index(0).Add(output_frame.release(),
|
if (sample_buffer_.num_samples() > 0 && pad_final_packet_) {
|
||||||
CurrentOutputTimestamp());
|
Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
|
||||||
|
¤t_timestamp_);
|
||||||
|
cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
|
||||||
|
.At(CurrentOutputTimestamp()));
|
||||||
}
|
}
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
@@ -258,7 +339,7 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
|
|||||||
cc->Inputs().Index(0).Header(), &input_header));
|
cc->Inputs().Index(0).Header(), &input_header));
|
||||||
|
|
||||||
sample_rate_ = input_header.sample_rate();
|
sample_rate_ = input_header.sample_rate();
|
||||||
num_channels_ = input_header.num_channels();
|
sample_buffer_.Init(sample_rate_, input_header.num_channels());
|
||||||
frame_duration_samples_ = time_series_util::SecondsToSamples(
|
frame_duration_samples_ = time_series_util::SecondsToSamples(
|
||||||
framer_options.frame_duration_seconds(), sample_rate_);
|
framer_options.frame_duration_seconds(), sample_rate_);
|
||||||
RET_CHECK_GT(frame_duration_samples_, 0)
|
RET_CHECK_GT(frame_duration_samples_, 0)
|
||||||
@@ -312,9 +393,8 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (use_window_) {
|
if (use_window_) {
|
||||||
window_ = Matrix::Ones(num_channels_, 1) *
|
window_ = Eigen::Map<Eigen::RowVectorXd>(window_vector.data(),
|
||||||
Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
|
frame_duration_samples_)
|
||||||
frame_duration_samples_)
|
|
||||||
.cast<float>();
|
.cast<float>();
|
||||||
}
|
}
|
||||||
use_local_timestamp_ = framer_options.use_local_timestamp();
|
use_local_timestamp_ = framer_options.use_local_timestamp();
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
// Copyright 2023 The MediaPipe Authors.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
//
|
||||||
|
// Benchmark for TimeSeriesFramerCalculator.
|
||||||
|
#include <memory>
|
||||||
|
#include <random>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
|
#include "benchmark/benchmark.h"
|
||||||
|
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
||||||
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
|
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||||
|
#include "mediapipe/framework/packet.h"
|
||||||
|
|
||||||
|
using ::mediapipe::Matrix;
|
||||||
|
|
||||||
|
void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
|
||||||
|
constexpr float kSampleRate = 32000.0;
|
||||||
|
constexpr int kNumChannels = 2;
|
||||||
|
constexpr int kFrameDurationSeconds = 5.0;
|
||||||
|
std::mt19937 rng(0 /*seed*/);
|
||||||
|
// Input around a half second's worth of samples at a time.
|
||||||
|
std::uniform_int_distribution<int> input_size_dist(15000, 17000);
|
||||||
|
// Generate a pool of random blocks of samples up front.
|
||||||
|
std::vector<Matrix> sample_pool;
|
||||||
|
sample_pool.reserve(20);
|
||||||
|
for (int i = 0; i < 20; ++i) {
|
||||||
|
sample_pool.push_back(Matrix::Random(kNumChannels, input_size_dist(rng)));
|
||||||
|
}
|
||||||
|
std::uniform_int_distribution<int> pool_index_dist(0, sample_pool.size() - 1);
|
||||||
|
|
||||||
|
mediapipe::CalculatorGraphConfig config;
|
||||||
|
config.add_input_stream("input");
|
||||||
|
config.add_output_stream("output");
|
||||||
|
auto* node = config.add_node();
|
||||||
|
node->set_calculator("TimeSeriesFramerCalculator");
|
||||||
|
node->add_input_stream("input");
|
||||||
|
node->add_output_stream("output");
|
||||||
|
mediapipe::TimeSeriesFramerCalculatorOptions* options =
|
||||||
|
node->mutable_options()->MutableExtension(
|
||||||
|
mediapipe::TimeSeriesFramerCalculatorOptions::ext);
|
||||||
|
options->set_frame_duration_seconds(kFrameDurationSeconds);
|
||||||
|
|
||||||
|
for (auto _ : state) {
|
||||||
|
state.PauseTiming(); // Pause benchmark timing.
|
||||||
|
|
||||||
|
// Prepare input packets of random blocks of samples.
|
||||||
|
std::vector<mediapipe::Packet> input_packets;
|
||||||
|
input_packets.reserve(32);
|
||||||
|
float t = 0;
|
||||||
|
for (int i = 0; i < 32; ++i) {
|
||||||
|
auto samples =
|
||||||
|
std::make_unique<Matrix>(sample_pool[pool_index_dist(rng)]);
|
||||||
|
const int num_samples = samples->cols();
|
||||||
|
input_packets.push_back(mediapipe::Adopt(samples.release())
|
||||||
|
.At(mediapipe::Timestamp::FromSeconds(t)));
|
||||||
|
t += num_samples / kSampleRate;
|
||||||
|
}
|
||||||
|
// Initialize graph.
|
||||||
|
mediapipe::CalculatorGraph graph;
|
||||||
|
ABSL_CHECK_OK(graph.Initialize(config));
|
||||||
|
// Prepare input header.
|
||||||
|
auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
|
||||||
|
header->set_sample_rate(kSampleRate);
|
||||||
|
header->set_num_channels(kNumChannels);
|
||||||
|
|
||||||
|
state.ResumeTiming(); // Resume benchmark timing.
|
||||||
|
|
||||||
|
ABSL_CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
|
||||||
|
for (auto& packet : input_packets) {
|
||||||
|
ABSL_CHECK_OK(graph.AddPacketToInputStream("input", packet));
|
||||||
|
}
|
||||||
|
ABSL_CHECK(!graph.HasError());
|
||||||
|
ABSL_CHECK_OK(graph.CloseAllInputStreams());
|
||||||
|
ABSL_CHECK_OK(graph.WaitUntilIdle());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BENCHMARK(BM_TimeSeriesFramerCalculator);
|
||||||
|
|
||||||
|
BENCHMARK_MAIN();
|
||||||
@@ -19,6 +19,7 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "audio/dsp/window_functions.h"
|
#include "audio/dsp/window_functions.h"
|
||||||
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
@@ -186,11 +187,12 @@ class TimeSeriesFramerCalculatorTest
|
|||||||
const int num_unique_output_samples =
|
const int num_unique_output_samples =
|
||||||
round((output().packets.size() - 1) * frame_step_samples) +
|
round((output().packets.size() - 1) * frame_step_samples) +
|
||||||
frame_duration_samples;
|
frame_duration_samples;
|
||||||
LOG(INFO) << "packets.size()=" << output().packets.size()
|
ABSL_LOG(INFO) << "packets.size()=" << output().packets.size()
|
||||||
<< " frame_duration_samples=" << frame_duration_samples
|
<< " frame_duration_samples=" << frame_duration_samples
|
||||||
<< " frame_step_samples=" << frame_step_samples
|
<< " frame_step_samples=" << frame_step_samples
|
||||||
<< " num_input_samples_=" << num_input_samples_
|
<< " num_input_samples_=" << num_input_samples_
|
||||||
<< " num_unique_output_samples=" << num_unique_output_samples;
|
<< " num_unique_output_samples="
|
||||||
|
<< num_unique_output_samples;
|
||||||
const int num_padding_samples =
|
const int num_padding_samples =
|
||||||
num_unique_output_samples - num_input_samples_;
|
num_unique_output_samples - num_input_samples_;
|
||||||
if (options_.pad_final_packet()) {
|
if (options_.pad_final_packet()) {
|
||||||
|
|||||||
@@ -117,6 +117,7 @@ mediapipe_proto_library(
|
|||||||
"//mediapipe/framework:calculator_proto",
|
"//mediapipe/framework:calculator_proto",
|
||||||
"//mediapipe/framework/formats:classification_proto",
|
"//mediapipe/framework/formats:classification_proto",
|
||||||
"//mediapipe/framework/formats:landmark_proto",
|
"//mediapipe/framework/formats:landmark_proto",
|
||||||
|
"//mediapipe/framework/formats:matrix_data_proto",
|
||||||
"//mediapipe/framework/formats:time_series_header_proto",
|
"//mediapipe/framework/formats:time_series_header_proto",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@@ -192,17 +193,19 @@ cc_library(
|
|||||||
"//mediapipe/framework:calculator_context",
|
"//mediapipe/framework:calculator_context",
|
||||||
"//mediapipe/framework:calculator_contract",
|
"//mediapipe/framework:calculator_contract",
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework:collection_item_id",
|
|
||||||
"//mediapipe/framework:packet",
|
"//mediapipe/framework:packet",
|
||||||
"//mediapipe/framework/formats:detection_cc_proto",
|
"//mediapipe/framework/formats:detection_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:image",
|
||||||
|
"//mediapipe/framework/formats:image_frame",
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
"//mediapipe/framework/formats:matrix",
|
"//mediapipe/framework/formats:matrix",
|
||||||
"//mediapipe/framework/formats:rect_cc_proto",
|
"//mediapipe/framework/formats:rect_cc_proto",
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"//mediapipe/framework/port:integral_types",
|
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
|
"//mediapipe/gpu:gpu_buffer",
|
||||||
"@com_google_absl//absl/memory",
|
"@com_google_absl//absl/memory",
|
||||||
|
"@com_google_absl//absl/status",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
@@ -215,18 +218,18 @@ cc_library(
|
|||||||
"//mediapipe/framework:calculator_context",
|
"//mediapipe/framework:calculator_context",
|
||||||
"//mediapipe/framework:calculator_contract",
|
"//mediapipe/framework:calculator_contract",
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework:collection_item_id",
|
|
||||||
"//mediapipe/framework/formats:classification_cc_proto",
|
"//mediapipe/framework/formats:classification_cc_proto",
|
||||||
"//mediapipe/framework/formats:detection_cc_proto",
|
"//mediapipe/framework/formats:detection_cc_proto",
|
||||||
"//mediapipe/framework/formats:image",
|
"//mediapipe/framework/formats:image",
|
||||||
|
"//mediapipe/framework/formats:image_frame",
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
"//mediapipe/framework/formats:matrix",
|
"//mediapipe/framework/formats:matrix",
|
||||||
"//mediapipe/framework/formats:rect_cc_proto",
|
"//mediapipe/framework/formats:rect_cc_proto",
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"//mediapipe/framework/port:integral_types",
|
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/gpu:gpu_buffer",
|
||||||
"//mediapipe/util:render_data_cc_proto",
|
"//mediapipe/util:render_data_cc_proto",
|
||||||
|
"@com_google_absl//absl/status",
|
||||||
"@org_tensorflow//tensorflow/lite:framework",
|
"@org_tensorflow//tensorflow/lite:framework",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
@@ -287,6 +290,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/api2:node",
|
"//mediapipe/framework/api2:node",
|
||||||
"//mediapipe/framework/api2:port",
|
"//mediapipe/framework/api2:port",
|
||||||
"//mediapipe/framework/formats:classification_cc_proto",
|
"//mediapipe/framework/formats:classification_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:image",
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
@@ -295,8 +299,7 @@ cc_library(
|
|||||||
"//mediapipe/util:render_data_cc_proto",
|
"//mediapipe/util:render_data_cc_proto",
|
||||||
"@org_tensorflow//tensorflow/lite:framework",
|
"@org_tensorflow//tensorflow/lite:framework",
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
":ios_or_disable_gpu": [],
|
||||||
"//mediapipe:ios": [],
|
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||||
],
|
],
|
||||||
@@ -322,6 +325,7 @@ cc_library(
|
|||||||
":concatenate_vector_calculator_cc_proto",
|
":concatenate_vector_calculator_cc_proto",
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/api2:node",
|
"//mediapipe/framework/api2:node",
|
||||||
|
"//mediapipe/framework/formats:body_rig_cc_proto",
|
||||||
"//mediapipe/framework/formats:classification_cc_proto",
|
"//mediapipe/framework/formats:classification_cc_proto",
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
@@ -378,17 +382,6 @@ cc_library(
|
|||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
cc_library(
|
|
||||||
name = "clip_detection_vector_size_calculator",
|
|
||||||
srcs = ["clip_detection_vector_size_calculator.cc"],
|
|
||||||
deps = [
|
|
||||||
":clip_vector_size_calculator",
|
|
||||||
"//mediapipe/framework:calculator_framework",
|
|
||||||
"//mediapipe/framework/formats:detection_cc_proto",
|
|
||||||
],
|
|
||||||
alwayslink = 1,
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_test(
|
cc_test(
|
||||||
name = "clip_vector_size_calculator_test",
|
name = "clip_vector_size_calculator_test",
|
||||||
srcs = ["clip_vector_size_calculator_test.cc"],
|
srcs = ["clip_vector_size_calculator_test.cc"],
|
||||||
@@ -590,6 +583,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:logging",
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/tool:options_util",
|
"//mediapipe/framework/tool:options_util",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
@@ -605,6 +599,7 @@ cc_test(
|
|||||||
"//mediapipe/framework/formats:video_stream_header",
|
"//mediapipe/framework/formats:video_stream_header",
|
||||||
"//mediapipe/framework/port:gtest_main",
|
"//mediapipe/framework/port:gtest_main",
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@@ -637,6 +632,7 @@ cc_library(
|
|||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
@@ -784,10 +780,11 @@ cc_library(
|
|||||||
"//mediapipe/framework/deps:random",
|
"//mediapipe/framework/deps:random",
|
||||||
"//mediapipe/framework/formats:video_stream_header",
|
"//mediapipe/framework/formats:video_stream_header",
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
"//mediapipe/framework/port:logging",
|
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/tool:options_util",
|
"//mediapipe/framework/tool:options_util",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
@@ -843,6 +840,7 @@ cc_test(
|
|||||||
"//mediapipe/framework/port:gtest_main",
|
"//mediapipe/framework/port:gtest_main",
|
||||||
"//mediapipe/framework/port:logging",
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/tool:validate_type",
|
"//mediapipe/framework/tool:validate_type",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@@ -904,6 +902,7 @@ cc_library(
|
|||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/formats:classification_cc_proto",
|
"//mediapipe/framework/formats:classification_cc_proto",
|
||||||
"//mediapipe/framework/formats:detection_cc_proto",
|
"//mediapipe/framework/formats:detection_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:image",
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
"//mediapipe/framework/formats:matrix",
|
"//mediapipe/framework/formats:matrix",
|
||||||
"//mediapipe/framework/formats:rect_cc_proto",
|
"//mediapipe/framework/formats:rect_cc_proto",
|
||||||
@@ -946,6 +945,7 @@ cc_library(
|
|||||||
deps = [
|
deps = [
|
||||||
":split_vector_calculator_cc_proto",
|
":split_vector_calculator_cc_proto",
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
|
"//mediapipe/framework/formats:body_rig_cc_proto",
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
@@ -1029,6 +1029,7 @@ cc_library(
|
|||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/api2:node",
|
"//mediapipe/framework/api2:node",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
@@ -1067,6 +1068,7 @@ cc_test(
|
|||||||
"//mediapipe/framework:calculator_runner",
|
"//mediapipe/framework:calculator_runner",
|
||||||
"//mediapipe/framework/port:gtest_main",
|
"//mediapipe/framework/port:gtest_main",
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
"//mediapipe/framework/port:parse_text_proto",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1113,6 +1115,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/api2:node",
|
"//mediapipe/framework/api2:node",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
@@ -1136,6 +1139,7 @@ cc_library(
|
|||||||
deps = [
|
deps = [
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework:timestamp",
|
"//mediapipe/framework:timestamp",
|
||||||
|
"//mediapipe/framework/api2:node",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
@@ -1164,6 +1168,7 @@ cc_library(
|
|||||||
"//mediapipe/framework:collection_item_id",
|
"//mediapipe/framework:collection_item_id",
|
||||||
"//mediapipe/framework/formats:classification_cc_proto",
|
"//mediapipe/framework/formats:classification_cc_proto",
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:matrix_data_cc_proto",
|
||||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
@@ -1238,6 +1243,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/formats:classification_cc_proto",
|
"//mediapipe/framework/formats:classification_cc_proto",
|
||||||
"//mediapipe/framework/formats:detection_cc_proto",
|
"//mediapipe/framework/formats:detection_cc_proto",
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:rect_cc_proto",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
],
|
],
|
||||||
@@ -1385,3 +1391,26 @@ cc_test(
|
|||||||
"@com_google_absl//absl/types:optional",
|
"@com_google_absl//absl/types:optional",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cc_library(
|
||||||
|
name = "value_or_default_calculator",
|
||||||
|
srcs = ["value_or_default_calculator.cc"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
deps = [
|
||||||
|
"//mediapipe/framework:calculator_framework",
|
||||||
|
"//mediapipe/framework/port:status",
|
||||||
|
],
|
||||||
|
alwayslink = True,
|
||||||
|
)
|
||||||
|
|
||||||
|
cc_test(
|
||||||
|
name = "value_or_default_calculator_test",
|
||||||
|
srcs = ["value_or_default_calculator_test.cc"],
|
||||||
|
deps = [
|
||||||
|
":value_or_default_calculator",
|
||||||
|
"//mediapipe/framework:calculator_framework",
|
||||||
|
"//mediapipe/framework:calculator_runner",
|
||||||
|
"//mediapipe/framework:packet",
|
||||||
|
"//mediapipe/framework/port:gtest_main",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/memory/memory.h"
|
#include "absl/memory/memory.h"
|
||||||
@@ -163,6 +164,75 @@ TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) {
|
|||||||
PacketOfIntsEq(input_timestamp2, std::vector<int>{3, 4})));
|
PacketOfIntsEq(input_timestamp2, std::vector<int>{3, 4})));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST(BeginEndLoopCalculatorPossibleDataRaceTest,
|
||||||
|
EndLoopForIntegersDoesNotRace) {
|
||||||
|
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
|
R"pb(
|
||||||
|
num_threads: 4
|
||||||
|
input_stream: "ints"
|
||||||
|
node {
|
||||||
|
calculator: "BeginLoopIntegerCalculator"
|
||||||
|
input_stream: "ITERABLE:ints"
|
||||||
|
output_stream: "ITEM:int"
|
||||||
|
output_stream: "BATCH_END:timestamp"
|
||||||
|
}
|
||||||
|
node {
|
||||||
|
calculator: "IncrementCalculator"
|
||||||
|
input_stream: "int"
|
||||||
|
output_stream: "int_plus_one"
|
||||||
|
}
|
||||||
|
# BEGIN: Data race possibility
|
||||||
|
# EndLoop###Calculator and another calculator using the same input
|
||||||
|
# may introduce race due to EndLoop###Calculator possibly consuming
|
||||||
|
# packet.
|
||||||
|
node {
|
||||||
|
calculator: "EndLoopIntegersCalculator"
|
||||||
|
input_stream: "ITEM:int_plus_one"
|
||||||
|
input_stream: "BATCH_END:timestamp"
|
||||||
|
output_stream: "ITERABLE:ints_plus_one"
|
||||||
|
}
|
||||||
|
node {
|
||||||
|
calculator: "IncrementCalculator"
|
||||||
|
input_stream: "int_plus_one"
|
||||||
|
output_stream: "int_plus_two"
|
||||||
|
}
|
||||||
|
# END: Data race possibility
|
||||||
|
node {
|
||||||
|
calculator: "EndLoopIntegersCalculator"
|
||||||
|
input_stream: "ITEM:int_plus_two"
|
||||||
|
input_stream: "BATCH_END:timestamp"
|
||||||
|
output_stream: "ITERABLE:ints_plus_two"
|
||||||
|
}
|
||||||
|
)pb");
|
||||||
|
std::vector<Packet> int_plus_one_packets;
|
||||||
|
tool::AddVectorSink("ints_plus_one", &graph_config, &int_plus_one_packets);
|
||||||
|
std::vector<Packet> int_original_packets;
|
||||||
|
tool::AddVectorSink("ints_plus_two", &graph_config, &int_original_packets);
|
||||||
|
|
||||||
|
CalculatorGraph graph;
|
||||||
|
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||||
|
MP_ASSERT_OK(graph.StartRun({}));
|
||||||
|
for (int i = 0; i < 100; ++i) {
|
||||||
|
std::vector<int> ints = {i, i + 1, i + 2};
|
||||||
|
Timestamp ts = Timestamp(i);
|
||||||
|
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||||
|
"ints", MakePacket<std::vector<int>>(std::move(ints)).At(ts)));
|
||||||
|
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||||
|
EXPECT_THAT(int_plus_one_packets,
|
||||||
|
testing::ElementsAre(
|
||||||
|
PacketOfIntsEq(ts, std::vector<int>{i + 1, i + 2, i + 3})));
|
||||||
|
EXPECT_THAT(int_original_packets,
|
||||||
|
testing::ElementsAre(
|
||||||
|
PacketOfIntsEq(ts, std::vector<int>{i + 2, i + 3, i + 4})));
|
||||||
|
|
||||||
|
int_plus_one_packets.clear();
|
||||||
|
int_original_packets.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
||||||
|
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||||
|
}
|
||||||
|
|
||||||
// Passes non empty vector through or outputs empty vector in case of timestamp
|
// Passes non empty vector through or outputs empty vector in case of timestamp
|
||||||
// bound update.
|
// bound update.
|
||||||
class PassThroughOrEmptyVectorCalculator : public CalculatorBase {
|
class PassThroughOrEmptyVectorCalculator : public CalculatorBase {
|
||||||
|
|||||||
@@ -17,10 +17,13 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "mediapipe/framework/formats/detection.pb.h"
|
#include "mediapipe/framework/formats/detection.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image.h"
|
||||||
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
#include "mediapipe/framework/formats/rect.pb.h"
|
#include "mediapipe/framework/formats/rect.pb.h"
|
||||||
#include "mediapipe/framework/formats/tensor.h"
|
#include "mediapipe/framework/formats/tensor.h"
|
||||||
|
#include "mediapipe/gpu/gpu_buffer.h"
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
|
|
||||||
@@ -60,4 +63,22 @@ REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
|
|||||||
typedef BeginLoopCalculator<std::vector<Tensor>> BeginLoopTensorCalculator;
|
typedef BeginLoopCalculator<std::vector<Tensor>> BeginLoopTensorCalculator;
|
||||||
REGISTER_CALCULATOR(BeginLoopTensorCalculator);
|
REGISTER_CALCULATOR(BeginLoopTensorCalculator);
|
||||||
|
|
||||||
|
// A calculator to process std::vector<mediapipe::ImageFrame>.
|
||||||
|
typedef BeginLoopCalculator<std::vector<ImageFrame>>
|
||||||
|
BeginLoopImageFrameCalculator;
|
||||||
|
REGISTER_CALCULATOR(BeginLoopImageFrameCalculator);
|
||||||
|
|
||||||
|
// A calculator to process std::vector<mediapipe::GpuBuffer>.
|
||||||
|
typedef BeginLoopCalculator<std::vector<GpuBuffer>>
|
||||||
|
BeginLoopGpuBufferCalculator;
|
||||||
|
REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator);
|
||||||
|
|
||||||
|
// A calculator to process std::vector<mediapipe::Image>.
|
||||||
|
typedef BeginLoopCalculator<std::vector<Image>> BeginLoopImageCalculator;
|
||||||
|
REGISTER_CALCULATOR(BeginLoopImageCalculator);
|
||||||
|
|
||||||
|
// A calculator to process std::vector<float>.
|
||||||
|
typedef BeginLoopCalculator<std::vector<float>> BeginLoopFloatCalculator;
|
||||||
|
REGISTER_CALCULATOR(BeginLoopFloatCalculator);
|
||||||
|
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|||||||
@@ -15,47 +15,57 @@
|
|||||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
||||||
#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
||||||
|
|
||||||
|
#include "absl/status/status.h"
|
||||||
#include "mediapipe/framework/calculator_context.h"
|
#include "mediapipe/framework/calculator_context.h"
|
||||||
#include "mediapipe/framework/calculator_contract.h"
|
#include "mediapipe/framework/calculator_contract.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/collection_item_id.h"
|
|
||||||
#include "mediapipe/framework/packet.h"
|
#include "mediapipe/framework/packet.h"
|
||||||
#include "mediapipe/framework/port/integral_types.h"
|
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
|
||||||
#include "mediapipe/framework/port/status_macros.h"
|
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
|
|
||||||
// Calculator for implementing loops on iterable collections inside a MediaPipe
|
// Calculator for implementing loops on iterable collections inside a MediaPipe
|
||||||
// graph.
|
// graph. Assume InputIterT is an iterable for type InputT, and OutputIterT is
|
||||||
|
// an iterable for type OutputT, e.g. vector<InputT> and vector<OutputT>.
|
||||||
|
// First, instantiate specializations in the loop calculators' implementations
|
||||||
|
// if missing:
|
||||||
|
// BeginLoopInputTCalculator = BeginLoopCalculator<InputIterT>
|
||||||
|
// EndLoopOutputTCalculator = EndLoopCalculator<OutputIterT>
|
||||||
|
// Then, the following graph transforms an item of type InputIterT to an
|
||||||
|
// OutputIterT by applying InputToOutputConverter to every element:
|
||||||
//
|
//
|
||||||
// It is designed to be used like:
|
// node { # Type @timestamp
|
||||||
//
|
// calculator: "BeginLoopInputTCalculator"
|
||||||
// node {
|
// input_stream: "ITERABLE:input_iterable" # InputIterT @iterable_ts
|
||||||
// calculator: "BeginLoopWithIterableCalculator"
|
// input_stream: "CLONE:extra_input" # ExtraT @extra_ts
|
||||||
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
|
// output_stream: "ITEM:input_iterator" # InputT @loop_internal_ts
|
||||||
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
|
// output_stream: "CLONE:cloned_extra_input" # ExtraT @loop_internal_ts
|
||||||
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
// output_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
|
||||||
// }
|
// }
|
||||||
//
|
//
|
||||||
// node {
|
// node {
|
||||||
// calculator: "ElementToBlaConverterSubgraph"
|
// calculator: "InputToOutputConverter"
|
||||||
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
|
// input_stream: "INPUT:input_iterator" # InputT @loop_internal_ts
|
||||||
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
|
// input_stream: "EXTRA:cloned_extra_input" # ExtraT @loop_internal_ts
|
||||||
|
// output_stream: "OUTPUT:output_iterator" # OutputT @loop_internal_ts
|
||||||
// }
|
// }
|
||||||
//
|
//
|
||||||
// node {
|
// node {
|
||||||
// calculator: "EndLoopWithOutputCalculator"
|
// calculator: "EndLoopOutputTCalculator"
|
||||||
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
|
// input_stream: "ITEM:output_iterator" # OutputT @loop_internal_ts
|
||||||
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
// input_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
|
||||||
// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
|
// output_stream: "ITERABLE:output_iterable" # OutputIterT @iterable_ts
|
||||||
// }
|
// }
|
||||||
//
|
//
|
||||||
|
// The resulting 'output_iterable' has the same timestamp as 'input_iterable'.
|
||||||
|
// The output packets of this calculator are part of the loop body and have
|
||||||
|
// loop-internal timestamps that are unrelated to the input iterator timestamp.
|
||||||
|
//
|
||||||
// Input streams tagged with "CLONE" are cloned to the corresponding output
|
// Input streams tagged with "CLONE" are cloned to the corresponding output
|
||||||
// streams at loop timestamps. This ensures that a MediaPipe graph or sub-graph
|
// streams at loop-internal timestamps. This ensures that a MediaPipe graph or
|
||||||
// can run multiple times, once per element in the "ITERABLE" for each pakcet
|
// sub-graph can run multiple times, once per element in the "ITERABLE" for each
|
||||||
// clone of the packets in the "CLONE" input streams.
|
// packet clone of the packets in the "CLONE" input streams. Think of CLONEd
|
||||||
|
// inputs as loop-wide constants.
|
||||||
template <typename IterableT>
|
template <typename IterableT>
|
||||||
class BeginLoopCalculator : public CalculatorBase {
|
class BeginLoopCalculator : public CalculatorBase {
|
||||||
using ItemT = typename IterableT::value_type;
|
using ItemT = typename IterableT::value_type;
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ class BypassCalculator : public Node {
|
|||||||
auto options = cc->Options<BypassCalculatorOptions>();
|
auto options = cc->Options<BypassCalculatorOptions>();
|
||||||
RET_CHECK_EQ(options.pass_input_stream().size(),
|
RET_CHECK_EQ(options.pass_input_stream().size(),
|
||||||
options.pass_output_stream().size());
|
options.pass_output_stream().size());
|
||||||
ASSIGN_OR_RETURN(
|
MP_ASSIGN_OR_RETURN(
|
||||||
auto pass_streams,
|
auto pass_streams,
|
||||||
GetPassMap(options, *cc->Inputs().TagMap(), *cc->Outputs().TagMap()));
|
GetPassMap(options, *cc->Inputs().TagMap(), *cc->Outputs().TagMap()));
|
||||||
std::set<CollectionItemId> pass_out;
|
std::set<CollectionItemId> pass_out;
|
||||||
@@ -121,8 +121,9 @@ class BypassCalculator : public Node {
|
|||||||
// Saves the map of passthrough input and output stream ids.
|
// Saves the map of passthrough input and output stream ids.
|
||||||
absl::Status Open(CalculatorContext* cc) override {
|
absl::Status Open(CalculatorContext* cc) override {
|
||||||
auto options = cc->Options<BypassCalculatorOptions>();
|
auto options = cc->Options<BypassCalculatorOptions>();
|
||||||
ASSIGN_OR_RETURN(pass_streams_, GetPassMap(options, *cc->Inputs().TagMap(),
|
MP_ASSIGN_OR_RETURN(
|
||||||
*cc->Outputs().TagMap()));
|
pass_streams_,
|
||||||
|
GetPassMap(options, *cc->Inputs().TagMap(), *cc->Outputs().TagMap()));
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
|
#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
|
||||||
#include "mediapipe/framework/api2/node.h"
|
#include "mediapipe/framework/api2/node.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/formats/body_rig.pb.h"
|
||||||
#include "mediapipe/framework/formats/classification.pb.h"
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
#include "mediapipe/framework/port/canonical_errors.h"
|
#include "mediapipe/framework/port/canonical_errors.h"
|
||||||
@@ -128,6 +129,19 @@ class ConcatenateClassificationListCalculator
|
|||||||
};
|
};
|
||||||
MEDIAPIPE_REGISTER_NODE(ConcatenateClassificationListCalculator);
|
MEDIAPIPE_REGISTER_NODE(ConcatenateClassificationListCalculator);
|
||||||
|
|
||||||
|
class ConcatenateJointListCalculator
|
||||||
|
: public ConcatenateListsCalculator<Joint, JointList> {
|
||||||
|
protected:
|
||||||
|
int ListSize(const JointList& list) const override {
|
||||||
|
return list.joint_size();
|
||||||
|
}
|
||||||
|
const Joint GetItem(const JointList& list, int idx) const override {
|
||||||
|
return list.joint(idx);
|
||||||
|
}
|
||||||
|
Joint* AddItem(JointList& list) const override { return list.add_joint(); }
|
||||||
|
};
|
||||||
|
MEDIAPIPE_REGISTER_NODE(ConcatenateJointListCalculator);
|
||||||
|
|
||||||
} // namespace api2
|
} // namespace api2
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "mediapipe/framework/formats/classification.pb.h"
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image.h"
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
#include "mediapipe/framework/formats/tensor.h"
|
#include "mediapipe/framework/formats/tensor.h"
|
||||||
#include "mediapipe/framework/port/integral_types.h"
|
#include "mediapipe/framework/port/integral_types.h"
|
||||||
@@ -55,6 +56,10 @@ MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator);
|
|||||||
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
|
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
|
||||||
MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator);
|
MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator);
|
||||||
|
|
||||||
|
typedef ConcatenateVectorCalculator<std::string>
|
||||||
|
ConcatenateStringVectorCalculator;
|
||||||
|
MEDIAPIPE_REGISTER_NODE(ConcatenateStringVectorCalculator);
|
||||||
|
|
||||||
// Example config:
|
// Example config:
|
||||||
// node {
|
// node {
|
||||||
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
|
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
|
||||||
@@ -100,4 +105,7 @@ typedef ConcatenateVectorCalculator<mediapipe::RenderData>
|
|||||||
ConcatenateRenderDataVectorCalculator;
|
ConcatenateRenderDataVectorCalculator;
|
||||||
MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);
|
MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);
|
||||||
|
|
||||||
|
typedef ConcatenateVectorCalculator<mediapipe::Image>
|
||||||
|
ConcatenateImageVectorCalculator;
|
||||||
|
MEDIAPIPE_REGISTER_NODE(ConcatenateImageVectorCalculator);
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|||||||
@@ -30,13 +30,15 @@ namespace mediapipe {
|
|||||||
typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
|
typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
|
||||||
MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
|
MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
|
||||||
|
|
||||||
void AddInputVector(int index, const std::vector<int>& input, int64_t timestamp,
|
template <typename T>
|
||||||
|
void AddInputVector(int index, const std::vector<T>& input, int64_t timestamp,
|
||||||
CalculatorRunner* runner) {
|
CalculatorRunner* runner) {
|
||||||
runner->MutableInputs()->Index(index).packets.push_back(
|
runner->MutableInputs()->Index(index).packets.push_back(
|
||||||
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
|
MakePacket<std::vector<T>>(input).At(Timestamp(timestamp)));
|
||||||
}
|
}
|
||||||
|
|
||||||
void AddInputVectors(const std::vector<std::vector<int>>& inputs,
|
template <typename T>
|
||||||
|
void AddInputVectors(const std::vector<std::vector<T>>& inputs,
|
||||||
int64_t timestamp, CalculatorRunner* runner) {
|
int64_t timestamp, CalculatorRunner* runner) {
|
||||||
for (int i = 0; i < inputs.size(); ++i) {
|
for (int i = 0; i < inputs.size(); ++i) {
|
||||||
AddInputVector(i, inputs[i], timestamp, runner);
|
AddInputVector(i, inputs[i], timestamp, runner);
|
||||||
@@ -382,6 +384,23 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
|||||||
EXPECT_EQ(0, outputs.size());
|
EXPECT_EQ(0, outputs.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST(ConcatenateStringVectorCalculatorTest, OneTimestamp) {
|
||||||
|
CalculatorRunner runner("ConcatenateStringVectorCalculator",
|
||||||
|
/*options_string=*/"", /*num_inputs=*/3,
|
||||||
|
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||||
|
|
||||||
|
std::vector<std::vector<std::string>> inputs = {
|
||||||
|
{"a", "b"}, {"c"}, {"d", "e", "f"}};
|
||||||
|
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||||
|
MP_ASSERT_OK(runner.Run());
|
||||||
|
|
||||||
|
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||||
|
EXPECT_EQ(1, outputs.size());
|
||||||
|
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||||
|
std::vector<std::string> expected_vector = {"a", "b", "c", "d", "e", "f"};
|
||||||
|
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<std::string>>());
|
||||||
|
}
|
||||||
|
|
||||||
typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
|
typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
|
||||||
TestConcatenateUniqueIntPtrCalculator;
|
TestConcatenateUniqueIntPtrCalculator;
|
||||||
MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator);
|
MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator);
|
||||||
|
|||||||
@@ -19,6 +19,7 @@
|
|||||||
#include "mediapipe/framework/collection_item_id.h"
|
#include "mediapipe/framework/collection_item_id.h"
|
||||||
#include "mediapipe/framework/formats/classification.pb.h"
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/matrix_data.pb.h"
|
||||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||||
#include "mediapipe/framework/port/canonical_errors.h"
|
#include "mediapipe/framework/port/canonical_errors.h"
|
||||||
#include "mediapipe/framework/port/integral_types.h"
|
#include "mediapipe/framework/port/integral_types.h"
|
||||||
@@ -78,15 +79,19 @@ class ConstantSidePacketCalculator : public CalculatorBase {
|
|||||||
} else if (packet_options.has_string_value()) {
|
} else if (packet_options.has_string_value()) {
|
||||||
packet.Set<std::string>();
|
packet.Set<std::string>();
|
||||||
} else if (packet_options.has_uint64_value()) {
|
} else if (packet_options.has_uint64_value()) {
|
||||||
packet.Set<uint64>();
|
packet.Set<uint64_t>();
|
||||||
} else if (packet_options.has_classification_list_value()) {
|
} else if (packet_options.has_classification_list_value()) {
|
||||||
packet.Set<ClassificationList>();
|
packet.Set<ClassificationList>();
|
||||||
} else if (packet_options.has_landmark_list_value()) {
|
} else if (packet_options.has_landmark_list_value()) {
|
||||||
packet.Set<LandmarkList>();
|
packet.Set<LandmarkList>();
|
||||||
} else if (packet_options.has_double_value()) {
|
} else if (packet_options.has_double_value()) {
|
||||||
packet.Set<double>();
|
packet.Set<double>();
|
||||||
|
} else if (packet_options.has_matrix_data_value()) {
|
||||||
|
packet.Set<MatrixData>();
|
||||||
} else if (packet_options.has_time_series_header_value()) {
|
} else if (packet_options.has_time_series_header_value()) {
|
||||||
packet.Set<TimeSeriesHeader>();
|
packet.Set<TimeSeriesHeader>();
|
||||||
|
} else if (packet_options.has_int64_value()) {
|
||||||
|
packet.Set<int64_t>();
|
||||||
} else {
|
} else {
|
||||||
return absl::InvalidArgumentError(
|
return absl::InvalidArgumentError(
|
||||||
"None of supported values were specified in options.");
|
"None of supported values were specified in options.");
|
||||||
@@ -112,7 +117,7 @@ class ConstantSidePacketCalculator : public CalculatorBase {
|
|||||||
} else if (packet_options.has_string_value()) {
|
} else if (packet_options.has_string_value()) {
|
||||||
packet.Set(MakePacket<std::string>(packet_options.string_value()));
|
packet.Set(MakePacket<std::string>(packet_options.string_value()));
|
||||||
} else if (packet_options.has_uint64_value()) {
|
} else if (packet_options.has_uint64_value()) {
|
||||||
packet.Set(MakePacket<uint64>(packet_options.uint64_value()));
|
packet.Set(MakePacket<uint64_t>(packet_options.uint64_value()));
|
||||||
} else if (packet_options.has_classification_list_value()) {
|
} else if (packet_options.has_classification_list_value()) {
|
||||||
packet.Set(MakePacket<ClassificationList>(
|
packet.Set(MakePacket<ClassificationList>(
|
||||||
packet_options.classification_list_value()));
|
packet_options.classification_list_value()));
|
||||||
@@ -121,9 +126,13 @@ class ConstantSidePacketCalculator : public CalculatorBase {
|
|||||||
MakePacket<LandmarkList>(packet_options.landmark_list_value()));
|
MakePacket<LandmarkList>(packet_options.landmark_list_value()));
|
||||||
} else if (packet_options.has_double_value()) {
|
} else if (packet_options.has_double_value()) {
|
||||||
packet.Set(MakePacket<double>(packet_options.double_value()));
|
packet.Set(MakePacket<double>(packet_options.double_value()));
|
||||||
|
} else if (packet_options.has_matrix_data_value()) {
|
||||||
|
packet.Set(MakePacket<MatrixData>(packet_options.matrix_data_value()));
|
||||||
} else if (packet_options.has_time_series_header_value()) {
|
} else if (packet_options.has_time_series_header_value()) {
|
||||||
packet.Set(MakePacket<TimeSeriesHeader>(
|
packet.Set(MakePacket<TimeSeriesHeader>(
|
||||||
packet_options.time_series_header_value()));
|
packet_options.time_series_header_value()));
|
||||||
|
} else if (packet_options.has_int64_value()) {
|
||||||
|
packet.Set(MakePacket<int64_t>(packet_options.int64_value()));
|
||||||
} else {
|
} else {
|
||||||
return absl::InvalidArgumentError(
|
return absl::InvalidArgumentError(
|
||||||
"None of supported values were specified in options.");
|
"None of supported values were specified in options.");
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ package mediapipe;
|
|||||||
import "mediapipe/framework/calculator.proto";
|
import "mediapipe/framework/calculator.proto";
|
||||||
import "mediapipe/framework/formats/classification.proto";
|
import "mediapipe/framework/formats/classification.proto";
|
||||||
import "mediapipe/framework/formats/landmark.proto";
|
import "mediapipe/framework/formats/landmark.proto";
|
||||||
|
import "mediapipe/framework/formats/matrix_data.proto";
|
||||||
import "mediapipe/framework/formats/time_series_header.proto";
|
import "mediapipe/framework/formats/time_series_header.proto";
|
||||||
|
|
||||||
message ConstantSidePacketCalculatorOptions {
|
message ConstantSidePacketCalculatorOptions {
|
||||||
@@ -29,14 +30,16 @@ message ConstantSidePacketCalculatorOptions {
|
|||||||
message ConstantSidePacket {
|
message ConstantSidePacket {
|
||||||
oneof value {
|
oneof value {
|
||||||
int32 int_value = 1;
|
int32 int_value = 1;
|
||||||
|
uint64 uint64_value = 5;
|
||||||
|
int64 int64_value = 11;
|
||||||
float float_value = 2;
|
float float_value = 2;
|
||||||
|
double double_value = 9;
|
||||||
bool bool_value = 3;
|
bool bool_value = 3;
|
||||||
string string_value = 4;
|
string string_value = 4;
|
||||||
uint64 uint64_value = 5;
|
|
||||||
ClassificationList classification_list_value = 6;
|
ClassificationList classification_list_value = 6;
|
||||||
LandmarkList landmark_list_value = 7;
|
LandmarkList landmark_list_value = 7;
|
||||||
double double_value = 9;
|
|
||||||
TimeSeriesHeader time_series_header_value = 10;
|
TimeSeriesHeader time_series_header_value = 10;
|
||||||
|
MatrixData matrix_data_value = 12;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include <cstdint>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "absl/strings/string_view.h"
|
#include "absl/strings/string_view.h"
|
||||||
@@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
|
|||||||
DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
|
DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
|
||||||
DoTestSingleSidePacket("{ bool_value: true }", true);
|
DoTestSingleSidePacket("{ bool_value: true }", true);
|
||||||
DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
|
DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
|
||||||
|
DoTestSingleSidePacket<int64_t>("{ int64_value: 63 }", 63);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
||||||
|
|||||||
@@ -14,15 +14,19 @@
|
|||||||
|
|
||||||
#include "mediapipe/calculators/core/end_loop_calculator.h"
|
#include "mediapipe/calculators/core/end_loop_calculator.h"
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <utility>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "mediapipe/framework/formats/classification.pb.h"
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
#include "mediapipe/framework/formats/detection.pb.h"
|
#include "mediapipe/framework/formats/detection.pb.h"
|
||||||
#include "mediapipe/framework/formats/image.h"
|
#include "mediapipe/framework/formats/image.h"
|
||||||
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
#include "mediapipe/framework/formats/rect.pb.h"
|
#include "mediapipe/framework/formats/rect.pb.h"
|
||||||
#include "mediapipe/framework/formats/tensor.h"
|
#include "mediapipe/framework/formats/tensor.h"
|
||||||
|
#include "mediapipe/gpu/gpu_buffer.h"
|
||||||
#include "mediapipe/util/render_data.pb.h"
|
#include "mediapipe/util/render_data.pb.h"
|
||||||
#include "tensorflow/lite/interpreter.h"
|
#include "tensorflow/lite/interpreter.h"
|
||||||
|
|
||||||
@@ -68,8 +72,22 @@ REGISTER_CALCULATOR(EndLoopMatrixCalculator);
|
|||||||
typedef EndLoopCalculator<std::vector<Tensor>> EndLoopTensorCalculator;
|
typedef EndLoopCalculator<std::vector<Tensor>> EndLoopTensorCalculator;
|
||||||
REGISTER_CALCULATOR(EndLoopTensorCalculator);
|
REGISTER_CALCULATOR(EndLoopTensorCalculator);
|
||||||
|
|
||||||
|
typedef EndLoopCalculator<std::vector<ImageFrame>> EndLoopImageFrameCalculator;
|
||||||
|
REGISTER_CALCULATOR(EndLoopImageFrameCalculator);
|
||||||
|
|
||||||
|
typedef EndLoopCalculator<std::vector<GpuBuffer>> EndLoopGpuBufferCalculator;
|
||||||
|
REGISTER_CALCULATOR(EndLoopGpuBufferCalculator);
|
||||||
|
|
||||||
typedef EndLoopCalculator<std::vector<::mediapipe::Image>>
|
typedef EndLoopCalculator<std::vector<::mediapipe::Image>>
|
||||||
EndLoopImageCalculator;
|
EndLoopImageCalculator;
|
||||||
REGISTER_CALCULATOR(EndLoopImageCalculator);
|
REGISTER_CALCULATOR(EndLoopImageCalculator);
|
||||||
|
|
||||||
|
typedef EndLoopCalculator<std::vector<std::array<float, 16>>>
|
||||||
|
EndLoopAffineMatrixCalculator;
|
||||||
|
REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
|
||||||
|
|
||||||
|
typedef EndLoopCalculator<std::vector<std::pair<int, int>>>
|
||||||
|
EndLoopImageSizeCalculator;
|
||||||
|
REGISTER_CALCULATOR(EndLoopImageSizeCalculator);
|
||||||
|
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|||||||
@@ -17,13 +17,11 @@
|
|||||||
|
|
||||||
#include <type_traits>
|
#include <type_traits>
|
||||||
|
|
||||||
|
#include "absl/status/status.h"
|
||||||
#include "mediapipe/framework/calculator_context.h"
|
#include "mediapipe/framework/calculator_context.h"
|
||||||
#include "mediapipe/framework/calculator_contract.h"
|
#include "mediapipe/framework/calculator_contract.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/collection_item_id.h"
|
|
||||||
#include "mediapipe/framework/port/integral_types.h"
|
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
|
|
||||||
@@ -33,27 +31,7 @@ namespace mediapipe {
|
|||||||
// from the "BATCH_END" tagged input stream, it emits the aggregated results
|
// from the "BATCH_END" tagged input stream, it emits the aggregated results
|
||||||
// at the original timestamp contained in the "BATCH_END" input stream.
|
// at the original timestamp contained in the "BATCH_END" input stream.
|
||||||
//
|
//
|
||||||
// It is designed to be used like:
|
// See BeginLoopCalculator for a usage example.
|
||||||
//
|
|
||||||
// node {
|
|
||||||
// calculator: "BeginLoopWithIterableCalculator"
|
|
||||||
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
|
|
||||||
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
|
|
||||||
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// node {
|
|
||||||
// calculator: "ElementToBlaConverterSubgraph"
|
|
||||||
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
|
|
||||||
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// node {
|
|
||||||
// calculator: "EndLoopWithOutputCalculator"
|
|
||||||
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
|
|
||||||
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
|
||||||
// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
|
|
||||||
// }
|
|
||||||
template <typename IterableT>
|
template <typename IterableT>
|
||||||
class EndLoopCalculator : public CalculatorBase {
|
class EndLoopCalculator : public CalculatorBase {
|
||||||
using ItemT = typename IterableT::value_type;
|
using ItemT = typename IterableT::value_type;
|
||||||
@@ -77,16 +55,16 @@ class EndLoopCalculator : public CalculatorBase {
|
|||||||
if (!input_stream_collection_) {
|
if (!input_stream_collection_) {
|
||||||
input_stream_collection_.reset(new IterableT);
|
input_stream_collection_.reset(new IterableT);
|
||||||
}
|
}
|
||||||
// Try to consume the item and move it into the collection. If the items
|
|
||||||
// are not consumable, then try to copy them instead. If the items are
|
if constexpr (std::is_copy_constructible_v<ItemT>) {
|
||||||
// not copiable, then an error will be returned.
|
input_stream_collection_->push_back(
|
||||||
auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
|
cc->Inputs().Tag("ITEM").Get<ItemT>());
|
||||||
if (item_ptr_or.ok()) {
|
|
||||||
input_stream_collection_->push_back(std::move(*item_ptr_or.value()));
|
|
||||||
} else {
|
} else {
|
||||||
if constexpr (std::is_copy_constructible_v<ItemT>) {
|
// Try to consume the item and move it into the collection. Return an
|
||||||
input_stream_collection_->push_back(
|
// error if the items are not consumable.
|
||||||
cc->Inputs().Tag("ITEM").template Get<ItemT>());
|
auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
|
||||||
|
if (item_ptr_or.ok()) {
|
||||||
|
input_stream_collection_->push_back(std::move(*item_ptr_or.value()));
|
||||||
} else {
|
} else {
|
||||||
return absl::InternalError(
|
return absl::InternalError(
|
||||||
"The item type is not copiable. Consider making the "
|
"The item type is not copiable. Consider making the "
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ constexpr char kOptionsTag[] = "OPTIONS";
|
|||||||
//
|
//
|
||||||
// Increasing `max_in_flight` to 2 or more can yield the better throughput
|
// Increasing `max_in_flight` to 2 or more can yield the better throughput
|
||||||
// when the graph exhibits a high degree of pipeline parallelism. Decreasing
|
// when the graph exhibits a high degree of pipeline parallelism. Decreasing
|
||||||
// `max_in_flight` to 0 can yield a better average latency, but at the cost of
|
// `max_in_queue` to 0 can yield a better average latency, but at the cost of
|
||||||
// lower throughput (lower framerate) due to the time during which the graph
|
// lower throughput (lower framerate) due to the time during which the graph
|
||||||
// is idle awaiting the next input frame.
|
// is idle awaiting the next input frame.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -26,19 +26,15 @@ constexpr char kStateChangeTag[] = "STATE_CHANGE";
|
|||||||
constexpr char kDisallowTag[] = "DISALLOW";
|
constexpr char kDisallowTag[] = "DISALLOW";
|
||||||
constexpr char kAllowTag[] = "ALLOW";
|
constexpr char kAllowTag[] = "ALLOW";
|
||||||
|
|
||||||
enum GateState {
|
std::string ToString(GateCalculatorOptions::GateState state) {
|
||||||
GATE_UNINITIALIZED,
|
|
||||||
GATE_ALLOW,
|
|
||||||
GATE_DISALLOW,
|
|
||||||
};
|
|
||||||
|
|
||||||
std::string ToString(GateState state) {
|
|
||||||
switch (state) {
|
switch (state) {
|
||||||
case GATE_UNINITIALIZED:
|
case GateCalculatorOptions::UNSPECIFIED:
|
||||||
|
return "UNSPECIFIED";
|
||||||
|
case GateCalculatorOptions::GATE_UNINITIALIZED:
|
||||||
return "UNINITIALIZED";
|
return "UNINITIALIZED";
|
||||||
case GATE_ALLOW:
|
case GateCalculatorOptions::GATE_ALLOW:
|
||||||
return "ALLOW";
|
return "ALLOW";
|
||||||
case GATE_DISALLOW:
|
case GateCalculatorOptions::GATE_DISALLOW:
|
||||||
return "DISALLOW";
|
return "DISALLOW";
|
||||||
}
|
}
|
||||||
DLOG(FATAL) << "Unknown GateState";
|
DLOG(FATAL) << "Unknown GateState";
|
||||||
@@ -153,10 +149,12 @@ class GateCalculator : public CalculatorBase {
|
|||||||
|
|
||||||
cc->SetOffset(TimestampDiff(0));
|
cc->SetOffset(TimestampDiff(0));
|
||||||
num_data_streams_ = cc->Inputs().NumEntries("");
|
num_data_streams_ = cc->Inputs().NumEntries("");
|
||||||
last_gate_state_ = GATE_UNINITIALIZED;
|
|
||||||
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
|
|
||||||
|
|
||||||
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
|
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
|
||||||
|
last_gate_state_ = options.initial_gate_state();
|
||||||
|
|
||||||
|
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
|
||||||
|
|
||||||
empty_packets_as_allow_ = options.empty_packets_as_allow();
|
empty_packets_as_allow_ = options.empty_packets_as_allow();
|
||||||
|
|
||||||
if (!use_side_packet_for_allow_disallow_ &&
|
if (!use_side_packet_for_allow_disallow_ &&
|
||||||
@@ -184,10 +182,12 @@ class GateCalculator : public CalculatorBase {
|
|||||||
allow = !cc->Inputs().Tag(kDisallowTag).Get<bool>();
|
allow = !cc->Inputs().Tag(kDisallowTag).Get<bool>();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
|
const GateCalculatorOptions::GateState new_gate_state =
|
||||||
|
allow ? GateCalculatorOptions::GATE_ALLOW
|
||||||
|
: GateCalculatorOptions::GATE_DISALLOW;
|
||||||
|
|
||||||
if (cc->Outputs().HasTag(kStateChangeTag)) {
|
if (cc->Outputs().HasTag(kStateChangeTag)) {
|
||||||
if (last_gate_state_ != GATE_UNINITIALIZED &&
|
if (last_gate_state_ != GateCalculatorOptions::GATE_UNINITIALIZED &&
|
||||||
last_gate_state_ != new_gate_state) {
|
last_gate_state_ != new_gate_state) {
|
||||||
VLOG(2) << "State transition in " << cc->NodeName() << " @ "
|
VLOG(2) << "State transition in " << cc->NodeName() << " @ "
|
||||||
<< cc->InputTimestamp().Value() << " from "
|
<< cc->InputTimestamp().Value() << " from "
|
||||||
@@ -223,7 +223,8 @@ class GateCalculator : public CalculatorBase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
GateState last_gate_state_ = GATE_UNINITIALIZED;
|
GateCalculatorOptions::GateState last_gate_state_ =
|
||||||
|
GateCalculatorOptions::GATE_UNINITIALIZED;
|
||||||
int num_data_streams_;
|
int num_data_streams_;
|
||||||
bool empty_packets_as_allow_;
|
bool empty_packets_as_allow_;
|
||||||
bool use_side_packet_for_allow_disallow_ = false;
|
bool use_side_packet_for_allow_disallow_ = false;
|
||||||
|
|||||||
@@ -31,4 +31,13 @@ message GateCalculatorOptions {
|
|||||||
// Whether to allow or disallow the input streams to pass when no
|
// Whether to allow or disallow the input streams to pass when no
|
||||||
// ALLOW/DISALLOW input or side input is specified.
|
// ALLOW/DISALLOW input or side input is specified.
|
||||||
optional bool allow = 2 [default = false];
|
optional bool allow = 2 [default = false];
|
||||||
|
|
||||||
|
enum GateState {
|
||||||
|
UNSPECIFIED = 0;
|
||||||
|
GATE_UNINITIALIZED = 1;
|
||||||
|
GATE_ALLOW = 2;
|
||||||
|
GATE_DISALLOW = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
optional GateState initial_gate_state = 3 [default = GATE_UNINITIALIZED];
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_runner.h"
|
#include "mediapipe/framework/calculator_runner.h"
|
||||||
#include "mediapipe/framework/port/gtest.h"
|
#include "mediapipe/framework/port/gtest.h"
|
||||||
@@ -35,14 +36,14 @@ class GateCalculatorTest : public ::testing::Test {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Use this when ALLOW/DISALLOW input is provided as a side packet.
|
// Use this when ALLOW/DISALLOW input is provided as a side packet.
|
||||||
void RunTimeStep(int64 timestamp, bool stream_payload) {
|
void RunTimeStep(int64_t timestamp, bool stream_payload) {
|
||||||
runner_->MutableInputs()->Get("", 0).packets.push_back(
|
runner_->MutableInputs()->Get("", 0).packets.push_back(
|
||||||
MakePacket<bool>(stream_payload).At(Timestamp(timestamp)));
|
MakePacket<bool>(stream_payload).At(Timestamp(timestamp)));
|
||||||
MP_ASSERT_OK(runner_->Run()) << "Calculator execution failed.";
|
MP_ASSERT_OK(runner_->Run()) << "Calculator execution failed.";
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use this when ALLOW/DISALLOW input is provided as an input stream.
|
// Use this when ALLOW/DISALLOW input is provided as an input stream.
|
||||||
void RunTimeStep(int64 timestamp, const std::string& control_tag,
|
void RunTimeStep(int64_t timestamp, const std::string& control_tag,
|
||||||
bool control) {
|
bool control) {
|
||||||
runner_->MutableInputs()->Get("", 0).packets.push_back(
|
runner_->MutableInputs()->Get("", 0).packets.push_back(
|
||||||
MakePacket<bool>(true).At(Timestamp(timestamp)));
|
MakePacket<bool>(true).At(Timestamp(timestamp)));
|
||||||
@@ -134,9 +135,9 @@ TEST_F(GateCalculatorTest, AllowByALLOWOptionToTrue) {
|
|||||||
}
|
}
|
||||||
)");
|
)");
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, true);
|
RunTimeStep(kTimestampValue0, true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, false);
|
RunTimeStep(kTimestampValue1, false);
|
||||||
|
|
||||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||||
@@ -159,9 +160,9 @@ TEST_F(GateCalculatorTest, DisallowByALLOWOptionSetToFalse) {
|
|||||||
}
|
}
|
||||||
)");
|
)");
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, true);
|
RunTimeStep(kTimestampValue0, true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, false);
|
RunTimeStep(kTimestampValue1, false);
|
||||||
|
|
||||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||||
@@ -175,9 +176,9 @@ TEST_F(GateCalculatorTest, DisallowByALLOWOptionNotSet) {
|
|||||||
output_stream: "test_output"
|
output_stream: "test_output"
|
||||||
)");
|
)");
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, true);
|
RunTimeStep(kTimestampValue0, true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, false);
|
RunTimeStep(kTimestampValue1, false);
|
||||||
|
|
||||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||||
@@ -193,9 +194,9 @@ TEST_F(GateCalculatorTest, AllowByALLOWSidePacketSetToTrue) {
|
|||||||
)");
|
)");
|
||||||
runner()->MutableSidePackets()->Tag(kAllowTag) = Adopt(new bool(true));
|
runner()->MutableSidePackets()->Tag(kAllowTag) = Adopt(new bool(true));
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, true);
|
RunTimeStep(kTimestampValue0, true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, false);
|
RunTimeStep(kTimestampValue1, false);
|
||||||
|
|
||||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||||
@@ -215,9 +216,9 @@ TEST_F(GateCalculatorTest, AllowByDisallowSidePacketSetToFalse) {
|
|||||||
)");
|
)");
|
||||||
runner()->MutableSidePackets()->Tag(kDisallowTag) = Adopt(new bool(false));
|
runner()->MutableSidePackets()->Tag(kDisallowTag) = Adopt(new bool(false));
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, true);
|
RunTimeStep(kTimestampValue0, true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, false);
|
RunTimeStep(kTimestampValue1, false);
|
||||||
|
|
||||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||||
@@ -237,9 +238,9 @@ TEST_F(GateCalculatorTest, DisallowByALLOWSidePacketSetToFalse) {
|
|||||||
)");
|
)");
|
||||||
runner()->MutableSidePackets()->Tag(kAllowTag) = Adopt(new bool(false));
|
runner()->MutableSidePackets()->Tag(kAllowTag) = Adopt(new bool(false));
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, true);
|
RunTimeStep(kTimestampValue0, true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, false);
|
RunTimeStep(kTimestampValue1, false);
|
||||||
|
|
||||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||||
@@ -255,9 +256,9 @@ TEST_F(GateCalculatorTest, DisallowByDISALLOWSidePacketSetToTrue) {
|
|||||||
)");
|
)");
|
||||||
runner()->MutableSidePackets()->Tag(kDisallowTag) = Adopt(new bool(true));
|
runner()->MutableSidePackets()->Tag(kDisallowTag) = Adopt(new bool(true));
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, true);
|
RunTimeStep(kTimestampValue0, true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, false);
|
RunTimeStep(kTimestampValue1, false);
|
||||||
|
|
||||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||||
@@ -272,13 +273,13 @@ TEST_F(GateCalculatorTest, Allow) {
|
|||||||
output_stream: "test_output"
|
output_stream: "test_output"
|
||||||
)");
|
)");
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, "ALLOW", true);
|
RunTimeStep(kTimestampValue0, "ALLOW", true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, "ALLOW", false);
|
RunTimeStep(kTimestampValue1, "ALLOW", false);
|
||||||
constexpr int64 kTimestampValue2 = 44;
|
constexpr int64_t kTimestampValue2 = 44;
|
||||||
RunTimeStep(kTimestampValue2, "ALLOW", true);
|
RunTimeStep(kTimestampValue2, "ALLOW", true);
|
||||||
constexpr int64 kTimestampValue3 = 45;
|
constexpr int64_t kTimestampValue3 = 45;
|
||||||
RunTimeStep(kTimestampValue3, "ALLOW", false);
|
RunTimeStep(kTimestampValue3, "ALLOW", false);
|
||||||
|
|
||||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||||
@@ -297,13 +298,13 @@ TEST_F(GateCalculatorTest, Disallow) {
|
|||||||
output_stream: "test_output"
|
output_stream: "test_output"
|
||||||
)");
|
)");
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, "DISALLOW", true);
|
RunTimeStep(kTimestampValue0, "DISALLOW", true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, "DISALLOW", false);
|
RunTimeStep(kTimestampValue1, "DISALLOW", false);
|
||||||
constexpr int64 kTimestampValue2 = 44;
|
constexpr int64_t kTimestampValue2 = 44;
|
||||||
RunTimeStep(kTimestampValue2, "DISALLOW", true);
|
RunTimeStep(kTimestampValue2, "DISALLOW", true);
|
||||||
constexpr int64 kTimestampValue3 = 45;
|
constexpr int64_t kTimestampValue3 = 45;
|
||||||
RunTimeStep(kTimestampValue3, "DISALLOW", false);
|
RunTimeStep(kTimestampValue3, "DISALLOW", false);
|
||||||
|
|
||||||
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
|
||||||
@@ -323,13 +324,13 @@ TEST_F(GateCalculatorTest, AllowWithStateChange) {
|
|||||||
output_stream: "STATE_CHANGE:state_changed"
|
output_stream: "STATE_CHANGE:state_changed"
|
||||||
)");
|
)");
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, "ALLOW", false);
|
RunTimeStep(kTimestampValue0, "ALLOW", false);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, "ALLOW", true);
|
RunTimeStep(kTimestampValue1, "ALLOW", true);
|
||||||
constexpr int64 kTimestampValue2 = 44;
|
constexpr int64_t kTimestampValue2 = 44;
|
||||||
RunTimeStep(kTimestampValue2, "ALLOW", true);
|
RunTimeStep(kTimestampValue2, "ALLOW", true);
|
||||||
constexpr int64 kTimestampValue3 = 45;
|
constexpr int64_t kTimestampValue3 = 45;
|
||||||
RunTimeStep(kTimestampValue3, "ALLOW", false);
|
RunTimeStep(kTimestampValue3, "ALLOW", false);
|
||||||
|
|
||||||
const std::vector<Packet>& output =
|
const std::vector<Packet>& output =
|
||||||
@@ -356,18 +357,18 @@ TEST_F(GateCalculatorTest, AllowWithStateChangeNoDataStreams) {
|
|||||||
RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true);
|
RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true);
|
||||||
constexpr int64_t kTimestampValue3 = 45;
|
constexpr int64_t kTimestampValue3 = 45;
|
||||||
RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false);
|
RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false);
|
||||||
LOG(INFO) << "a";
|
ABSL_LOG(INFO) << "a";
|
||||||
const std::vector<Packet>& output =
|
const std::vector<Packet>& output =
|
||||||
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
||||||
LOG(INFO) << "s";
|
ABSL_LOG(INFO) << "s";
|
||||||
ASSERT_EQ(2, output.size());
|
ASSERT_EQ(2, output.size());
|
||||||
LOG(INFO) << "d";
|
ABSL_LOG(INFO) << "d";
|
||||||
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
|
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
|
||||||
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
|
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
|
||||||
LOG(INFO) << "f";
|
ABSL_LOG(INFO) << "f";
|
||||||
EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
|
EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
|
||||||
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
|
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
|
||||||
LOG(INFO) << "g";
|
ABSL_LOG(INFO) << "g";
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(GateCalculatorTest, DisallowWithStateChange) {
|
TEST_F(GateCalculatorTest, DisallowWithStateChange) {
|
||||||
@@ -379,13 +380,13 @@ TEST_F(GateCalculatorTest, DisallowWithStateChange) {
|
|||||||
output_stream: "STATE_CHANGE:state_changed"
|
output_stream: "STATE_CHANGE:state_changed"
|
||||||
)");
|
)");
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, "DISALLOW", true);
|
RunTimeStep(kTimestampValue0, "DISALLOW", true);
|
||||||
constexpr int64 kTimestampValue1 = 43;
|
constexpr int64_t kTimestampValue1 = 43;
|
||||||
RunTimeStep(kTimestampValue1, "DISALLOW", false);
|
RunTimeStep(kTimestampValue1, "DISALLOW", false);
|
||||||
constexpr int64 kTimestampValue2 = 44;
|
constexpr int64_t kTimestampValue2 = 44;
|
||||||
RunTimeStep(kTimestampValue2, "DISALLOW", false);
|
RunTimeStep(kTimestampValue2, "DISALLOW", false);
|
||||||
constexpr int64 kTimestampValue3 = 45;
|
constexpr int64_t kTimestampValue3 = 45;
|
||||||
RunTimeStep(kTimestampValue3, "DISALLOW", true);
|
RunTimeStep(kTimestampValue3, "DISALLOW", true);
|
||||||
|
|
||||||
const std::vector<Packet>& output =
|
const std::vector<Packet>& output =
|
||||||
@@ -432,7 +433,7 @@ TEST_F(GateCalculatorTest, DisallowInitialNoStateTransition) {
|
|||||||
output_stream: "STATE_CHANGE:state_changed"
|
output_stream: "STATE_CHANGE:state_changed"
|
||||||
)");
|
)");
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, "DISALLOW", false);
|
RunTimeStep(kTimestampValue0, "DISALLOW", false);
|
||||||
|
|
||||||
const std::vector<Packet>& output =
|
const std::vector<Packet>& output =
|
||||||
@@ -450,7 +451,7 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
|
|||||||
output_stream: "STATE_CHANGE:state_changed"
|
output_stream: "STATE_CHANGE:state_changed"
|
||||||
)");
|
)");
|
||||||
|
|
||||||
constexpr int64 kTimestampValue0 = 42;
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
RunTimeStep(kTimestampValue0, "ALLOW", true);
|
RunTimeStep(kTimestampValue0, "ALLOW", true);
|
||||||
|
|
||||||
const std::vector<Packet>& output =
|
const std::vector<Packet>& output =
|
||||||
@@ -458,5 +459,29 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
|
|||||||
ASSERT_EQ(0, output.size());
|
ASSERT_EQ(0, output.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Must detect allow value for first timestamp as a state change when the
|
||||||
|
// initial state is set to GATE_DISALLOW.
|
||||||
|
TEST_F(GateCalculatorTest, StateChangeTriggeredWithInitialGateStateOption) {
|
||||||
|
SetRunner(R"(
|
||||||
|
calculator: "GateCalculator"
|
||||||
|
input_stream: "test_input"
|
||||||
|
input_stream: "ALLOW:allow"
|
||||||
|
output_stream: "test_output"
|
||||||
|
output_stream: "STATE_CHANGE:state_change"
|
||||||
|
options: {
|
||||||
|
[mediapipe.GateCalculatorOptions.ext] {
|
||||||
|
initial_gate_state: GATE_DISALLOW
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)");
|
||||||
|
|
||||||
|
constexpr int64_t kTimestampValue0 = 42;
|
||||||
|
RunTimeStep(kTimestampValue0, "ALLOW", true);
|
||||||
|
|
||||||
|
const std::vector<Packet>& output =
|
||||||
|
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
||||||
|
ASSERT_EQ(1, output.size());
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
#include "mediapipe/framework/formats/classification.pb.h"
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
#include "mediapipe/framework/formats/detection.pb.h"
|
#include "mediapipe/framework/formats/detection.pb.h"
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/rect.pb.h"
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
namespace api2 {
|
namespace api2 {
|
||||||
@@ -37,5 +38,12 @@ using GetDetectionVectorItemCalculator =
|
|||||||
GetVectorItemCalculator<mediapipe::Detection>;
|
GetVectorItemCalculator<mediapipe::Detection>;
|
||||||
REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
|
REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
|
||||||
|
|
||||||
|
using GetNormalizedRectVectorItemCalculator =
|
||||||
|
GetVectorItemCalculator<NormalizedRect>;
|
||||||
|
REGISTER_CALCULATOR(GetNormalizedRectVectorItemCalculator);
|
||||||
|
|
||||||
|
using GetRectVectorItemCalculator = GetVectorItemCalculator<Rect>;
|
||||||
|
REGISTER_CALCULATOR(GetRectVectorItemCalculator);
|
||||||
|
|
||||||
} // namespace api2
|
} // namespace api2
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
@@ -78,7 +79,7 @@ absl::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
|
|||||||
if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) {
|
if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) {
|
||||||
cc->Outputs().Index(0).AddPacket(packet);
|
cc->Outputs().Index(0).AddPacket(packet);
|
||||||
} else {
|
} else {
|
||||||
LOG_FIRST_N(WARNING, 5)
|
ABSL_LOG_FIRST_N(WARNING, 5)
|
||||||
<< "Dropping a packet with timestamp " << packet.Timestamp();
|
<< "Dropping a packet with timestamp " << packet.Timestamp();
|
||||||
}
|
}
|
||||||
if (cc->Outputs().NumEntries() >= 2) {
|
if (cc->Outputs().NumEntries() >= 2) {
|
||||||
|
|||||||
@@ -16,6 +16,7 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_runner.h"
|
#include "mediapipe/framework/calculator_runner.h"
|
||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
@@ -209,7 +210,7 @@ TEST(MatrixMultiplyCalculatorTest, Multiply) {
|
|||||||
MatrixFromTextProto(kSamplesText, &samples);
|
MatrixFromTextProto(kSamplesText, &samples);
|
||||||
Matrix expected;
|
Matrix expected;
|
||||||
MatrixFromTextProto(kExpectedText, &expected);
|
MatrixFromTextProto(kExpectedText, &expected);
|
||||||
CHECK_EQ(samples.cols(), expected.cols());
|
ABSL_CHECK_EQ(samples.cols(), expected.cols());
|
||||||
|
|
||||||
for (int i = 0; i < samples.cols(); ++i) {
|
for (int i = 0; i < samples.cols(); ++i) {
|
||||||
// Take a column from samples and produce a packet with just that
|
// Take a column from samples and produce a packet with just that
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ class MatrixToVectorCalculatorTest
|
|||||||
void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
|
void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
|
||||||
|
|
||||||
void AppendInput(const std::vector<float>& column_major_data,
|
void AppendInput(const std::vector<float>& column_major_data,
|
||||||
int64 timestamp) {
|
int64_t timestamp) {
|
||||||
ASSERT_EQ(num_input_samples_ * num_input_channels_,
|
ASSERT_EQ(num_input_samples_ * num_input_channels_,
|
||||||
column_major_data.size());
|
column_major_data.size());
|
||||||
Eigen::Map<const Matrix> data_map(&column_major_data[0],
|
Eigen::Map<const Matrix> data_map(&column_major_data[0],
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "mediapipe/framework/api2/node.h"
|
#include "mediapipe/framework/api2/node.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
@@ -53,7 +54,7 @@ class MergeCalculator : public Node {
|
|||||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||||
RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
|
RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
|
||||||
if (kIn(cc).Count() == 1) {
|
if (kIn(cc).Count() == 1) {
|
||||||
LOG(WARNING)
|
ABSL_LOG(WARNING)
|
||||||
<< "MergeCalculator expects multiple input streams to merge but is "
|
<< "MergeCalculator expects multiple input streams to merge but is "
|
||||||
"receiving only one. Make sure the calculator is configured "
|
"receiving only one. Make sure the calculator is configured "
|
||||||
"correctly or consider removing this calculator to reduce "
|
"correctly or consider removing this calculator to reduce "
|
||||||
@@ -72,8 +73,8 @@ class MergeCalculator : public Node {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
LOG(WARNING) << "Empty input packets at timestamp "
|
ABSL_LOG(WARNING) << "Empty input packets at timestamp "
|
||||||
<< cc->InputTimestamp().Value();
|
<< cc->InputTimestamp().Value();
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
/* Copyright 2022 The MediaPipe Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
/* Copyright 2022 The MediaPipe Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|||||||
@@ -16,9 +16,12 @@
|
|||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
// Reflect an integer against the lower and upper bound of an interval.
|
// Reflect an integer against the lower and upper bound of an interval.
|
||||||
int64 ReflectBetween(int64 ts, int64 ts_min, int64 ts_max) {
|
int64_t ReflectBetween(int64_t ts, int64_t ts_min, int64_t ts_max) {
|
||||||
if (ts < ts_min) return 2 * ts_min - ts - 1;
|
if (ts < ts_min) return 2 * ts_min - ts - 1;
|
||||||
if (ts >= ts_max) return 2 * ts_max - ts - 1;
|
if (ts >= ts_max) return 2 * ts_max - ts - 1;
|
||||||
return ts;
|
return ts;
|
||||||
@@ -47,7 +50,7 @@ constexpr char kOptionsTag[] = "OPTIONS";
|
|||||||
// Returns a TimestampDiff (assuming microseconds) corresponding to the
|
// Returns a TimestampDiff (assuming microseconds) corresponding to the
|
||||||
// given time in seconds.
|
// given time in seconds.
|
||||||
TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
TimestampDiff TimestampDiffFromSeconds(double seconds) {
|
||||||
return TimestampDiff(MathUtil::SafeRound<int64, double>(
|
return TimestampDiff(MathUtil::SafeRound<int64_t, double>(
|
||||||
seconds * Timestamp::kTimestampUnitsPerSecond));
|
seconds * Timestamp::kTimestampUnitsPerSecond));
|
||||||
}
|
}
|
||||||
} // namespace
|
} // namespace
|
||||||
@@ -117,8 +120,8 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
|
|||||||
<< "The output frame rate must be smaller than "
|
<< "The output frame rate must be smaller than "
|
||||||
<< Timestamp::kTimestampUnitsPerSecond;
|
<< Timestamp::kTimestampUnitsPerSecond;
|
||||||
|
|
||||||
frame_time_usec_ = static_cast<int64>(1000000.0 / frame_rate_);
|
frame_time_usec_ = static_cast<int64_t>(1000000.0 / frame_rate_);
|
||||||
jitter_usec_ = static_cast<int64>(1000000.0 * jitter_ / frame_rate_);
|
jitter_usec_ = static_cast<int64_t>(1000000.0 * jitter_ / frame_rate_);
|
||||||
RET_CHECK_LE(jitter_usec_, frame_time_usec_);
|
RET_CHECK_LE(jitter_usec_, frame_time_usec_);
|
||||||
|
|
||||||
video_header_.frame_rate = frame_rate_;
|
video_header_.frame_rate = frame_rate_;
|
||||||
@@ -177,7 +180,7 @@ PacketResamplerCalculator::GetSamplingStrategy(
|
|||||||
const PacketResamplerCalculatorOptions& options) {
|
const PacketResamplerCalculatorOptions& options) {
|
||||||
if (options.reproducible_sampling()) {
|
if (options.reproducible_sampling()) {
|
||||||
if (!options.jitter_with_reflection()) {
|
if (!options.jitter_with_reflection()) {
|
||||||
LOG(WARNING)
|
ABSL_LOG(WARNING)
|
||||||
<< "reproducible_sampling enabled w/ jitter_with_reflection "
|
<< "reproducible_sampling enabled w/ jitter_with_reflection "
|
||||||
"disabled. "
|
"disabled. "
|
||||||
<< "reproducible_sampling always uses jitter with reflection, "
|
<< "reproducible_sampling always uses jitter with reflection, "
|
||||||
@@ -198,17 +201,18 @@ PacketResamplerCalculator::GetSamplingStrategy(
|
|||||||
return absl::make_unique<JitterWithoutReflectionStrategy>(this);
|
return absl::make_unique<JitterWithoutReflectionStrategy>(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const {
|
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(
|
||||||
CHECK_EQ(jitter_, 0.0);
|
int64_t index) const {
|
||||||
CHECK_NE(first_timestamp_, Timestamp::Unset());
|
ABSL_CHECK_EQ(jitter_, 0.0);
|
||||||
|
ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset());
|
||||||
return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_);
|
return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_);
|
||||||
}
|
}
|
||||||
|
|
||||||
int64 PacketResamplerCalculator::TimestampToPeriodIndex(
|
int64_t PacketResamplerCalculator::TimestampToPeriodIndex(
|
||||||
Timestamp timestamp) const {
|
Timestamp timestamp) const {
|
||||||
CHECK_EQ(jitter_, 0.0);
|
ABSL_CHECK_EQ(jitter_, 0.0);
|
||||||
CHECK_NE(first_timestamp_, Timestamp::Unset());
|
ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset());
|
||||||
return MathUtil::SafeRound<int64, double>(
|
return MathUtil::SafeRound<int64_t, double>(
|
||||||
(timestamp - first_timestamp_).Seconds() * frame_rate_);
|
(timestamp - first_timestamp_).Seconds() * frame_rate_);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -228,13 +232,15 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
|
|||||||
|
|
||||||
if (resampler_options.output_header() !=
|
if (resampler_options.output_header() !=
|
||||||
PacketResamplerCalculatorOptions::NONE) {
|
PacketResamplerCalculatorOptions::NONE) {
|
||||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
ABSL_LOG(WARNING)
|
||||||
"the actual value.";
|
<< "VideoHeader::frame_rate holds the target value and not "
|
||||||
|
"the actual value.";
|
||||||
}
|
}
|
||||||
|
|
||||||
if (calculator_->flush_last_packet_) {
|
if (calculator_->flush_last_packet_) {
|
||||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
ABSL_LOG(WARNING)
|
||||||
"ignored, because we are adding jitter.";
|
<< "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||||
|
"ignored, because we are adding jitter.";
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
||||||
@@ -253,7 +259,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
|
|||||||
}
|
}
|
||||||
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
|
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
|
||||||
if (!packet_reservoir_->IsEmpty()) {
|
if (!packet_reservoir_->IsEmpty()) {
|
||||||
LOG(INFO) << "Emitting pack from reservoir.";
|
ABSL_LOG(INFO) << "Emitting pack from reservoir.";
|
||||||
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
||||||
}
|
}
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
@@ -284,16 +290,16 @@ absl::Status LegacyJitterWithReflectionStrategy::Process(
|
|||||||
|
|
||||||
if (calculator_->frame_time_usec_ <
|
if (calculator_->frame_time_usec_ <
|
||||||
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
||||||
LOG_FIRST_N(WARNING, 2)
|
ABSL_LOG_FIRST_N(WARNING, 2)
|
||||||
<< "Adding jitter is not very useful when upsampling.";
|
<< "Adding jitter is not very useful when upsampling.";
|
||||||
}
|
}
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
const int64 last_diff =
|
const int64_t last_diff =
|
||||||
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
|
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
|
||||||
.Value();
|
.Value();
|
||||||
RET_CHECK_GT(last_diff, 0);
|
RET_CHECK_GT(last_diff, 0);
|
||||||
const int64 curr_diff =
|
const int64_t curr_diff =
|
||||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||||
if (curr_diff > 0) {
|
if (curr_diff > 0) {
|
||||||
break;
|
break;
|
||||||
@@ -339,8 +345,8 @@ void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
|
|||||||
next_output_timestamp_ = Timestamp(ReflectBetween(
|
next_output_timestamp_ = Timestamp(ReflectBetween(
|
||||||
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
||||||
next_output_timestamp_max_.Value()));
|
next_output_timestamp_max_.Value()));
|
||||||
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
|
ABSL_CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
|
||||||
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
|
ABSL_CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status ReproducibleJitterWithReflectionStrategy::Open(
|
absl::Status ReproducibleJitterWithReflectionStrategy::Open(
|
||||||
@@ -351,13 +357,15 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Open(
|
|||||||
|
|
||||||
if (resampler_options.output_header() !=
|
if (resampler_options.output_header() !=
|
||||||
PacketResamplerCalculatorOptions::NONE) {
|
PacketResamplerCalculatorOptions::NONE) {
|
||||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
ABSL_LOG(WARNING)
|
||||||
"the actual value.";
|
<< "VideoHeader::frame_rate holds the target value and not "
|
||||||
|
"the actual value.";
|
||||||
}
|
}
|
||||||
|
|
||||||
if (calculator_->flush_last_packet_) {
|
if (calculator_->flush_last_packet_) {
|
||||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
ABSL_LOG(WARNING)
|
||||||
"ignored, because we are adding jitter.";
|
<< "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||||
|
"ignored, because we are adding jitter.";
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
||||||
@@ -410,7 +418,7 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Process(
|
|||||||
// Note, if the stream is upsampling, this could lead to the same packet
|
// Note, if the stream is upsampling, this could lead to the same packet
|
||||||
// being emitted twice. Upsampling and jitter doesn't make much sense
|
// being emitted twice. Upsampling and jitter doesn't make much sense
|
||||||
// but does technically work.
|
// but does technically work.
|
||||||
LOG_FIRST_N(WARNING, 2)
|
ABSL_LOG_FIRST_N(WARNING, 2)
|
||||||
<< "Adding jitter is not very useful when upsampling.";
|
<< "Adding jitter is not very useful when upsampling.";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -498,13 +506,15 @@ absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
|
|||||||
|
|
||||||
if (resampler_options.output_header() !=
|
if (resampler_options.output_header() !=
|
||||||
PacketResamplerCalculatorOptions::NONE) {
|
PacketResamplerCalculatorOptions::NONE) {
|
||||||
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
ABSL_LOG(WARNING)
|
||||||
"the actual value.";
|
<< "VideoHeader::frame_rate holds the target value and not "
|
||||||
|
"the actual value.";
|
||||||
}
|
}
|
||||||
|
|
||||||
if (calculator_->flush_last_packet_) {
|
if (calculator_->flush_last_packet_) {
|
||||||
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
ABSL_LOG(WARNING)
|
||||||
"ignored, because we are adding jitter.";
|
<< "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||||
|
"ignored, because we are adding jitter.";
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
||||||
@@ -554,16 +564,16 @@ absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
|
|||||||
|
|
||||||
if (calculator_->frame_time_usec_ <
|
if (calculator_->frame_time_usec_ <
|
||||||
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
||||||
LOG_FIRST_N(WARNING, 2)
|
ABSL_LOG_FIRST_N(WARNING, 2)
|
||||||
<< "Adding jitter is not very useful when upsampling.";
|
<< "Adding jitter is not very useful when upsampling.";
|
||||||
}
|
}
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
const int64 last_diff =
|
const int64_t last_diff =
|
||||||
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
|
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
|
||||||
.Value();
|
.Value();
|
||||||
RET_CHECK_GT(last_diff, 0);
|
RET_CHECK_GT(last_diff, 0);
|
||||||
const int64 curr_diff =
|
const int64_t curr_diff =
|
||||||
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
(next_output_timestamp_ - cc->InputTimestamp()).Value();
|
||||||
if (curr_diff > 0) {
|
if (curr_diff > 0) {
|
||||||
break;
|
break;
|
||||||
@@ -631,7 +641,7 @@ absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
|
|||||||
} else {
|
} else {
|
||||||
// Initialize first_timestamp_ with the first packet timestamp
|
// Initialize first_timestamp_ with the first packet timestamp
|
||||||
// aligned to the base_timestamp_.
|
// aligned to the base_timestamp_.
|
||||||
int64 first_index = MathUtil::SafeRound<int64, double>(
|
int64_t first_index = MathUtil::SafeRound<int64_t, double>(
|
||||||
(cc->InputTimestamp() - base_timestamp_).Seconds() *
|
(cc->InputTimestamp() - base_timestamp_).Seconds() *
|
||||||
calculator_->frame_rate_);
|
calculator_->frame_rate_);
|
||||||
calculator_->first_timestamp_ =
|
calculator_->first_timestamp_ =
|
||||||
@@ -646,7 +656,7 @@ absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
const Timestamp received_timestamp = cc->InputTimestamp();
|
const Timestamp received_timestamp = cc->InputTimestamp();
|
||||||
const int64 received_timestamp_idx =
|
const int64_t received_timestamp_idx =
|
||||||
calculator_->TimestampToPeriodIndex(received_timestamp);
|
calculator_->TimestampToPeriodIndex(received_timestamp);
|
||||||
// Only consider the received packet if it belongs to the current period
|
// Only consider the received packet if it belongs to the current period
|
||||||
// (== period_count_) or to a newer one (> period_count_).
|
// (== period_count_) or to a newer one (> period_count_).
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
#include "mediapipe/framework/deps/random_base.h"
|
#include "mediapipe/framework/deps/random_base.h"
|
||||||
#include "mediapipe/framework/formats/video_stream_header.h"
|
#include "mediapipe/framework/formats/video_stream_header.h"
|
||||||
#include "mediapipe/framework/port/integral_types.h"
|
#include "mediapipe/framework/port/integral_types.h"
|
||||||
#include "mediapipe/framework/port/logging.h"
|
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
#include "mediapipe/framework/port/status_macros.h"
|
#include "mediapipe/framework/port/status_macros.h"
|
||||||
|
|||||||
@@ -51,9 +51,9 @@ class SimpleRunner : public CalculatorRunner {
|
|||||||
|
|
||||||
virtual ~SimpleRunner() {}
|
virtual ~SimpleRunner() {}
|
||||||
|
|
||||||
void SetInput(const std::vector<int64>& timestamp_list) {
|
void SetInput(const std::vector<int64_t>& timestamp_list) {
|
||||||
MutableInputs()->Index(0).packets.clear();
|
MutableInputs()->Index(0).packets.clear();
|
||||||
for (const int64 ts : timestamp_list) {
|
for (const int64_t ts : timestamp_list) {
|
||||||
MutableInputs()->Index(0).packets.push_back(
|
MutableInputs()->Index(0).packets.push_back(
|
||||||
Adopt(new std::string(absl::StrCat("Frame #", ts)))
|
Adopt(new std::string(absl::StrCat("Frame #", ts)))
|
||||||
.At(Timestamp(ts)));
|
.At(Timestamp(ts)));
|
||||||
@@ -72,8 +72,8 @@ class SimpleRunner : public CalculatorRunner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void CheckOutputTimestamps(
|
void CheckOutputTimestamps(
|
||||||
const std::vector<int64>& expected_frames,
|
const std::vector<int64_t>& expected_frames,
|
||||||
const std::vector<int64>& expected_timestamps) const {
|
const std::vector<int64_t>& expected_timestamps) const {
|
||||||
EXPECT_EQ(expected_frames.size(), Outputs().Index(0).packets.size());
|
EXPECT_EQ(expected_frames.size(), Outputs().Index(0).packets.size());
|
||||||
EXPECT_EQ(expected_timestamps.size(), Outputs().Index(0).packets.size());
|
EXPECT_EQ(expected_timestamps.size(), Outputs().Index(0).packets.size());
|
||||||
int count = 0;
|
int count = 0;
|
||||||
@@ -112,7 +112,7 @@ MATCHER_P2(PacketAtTimestamp, payload, timestamp,
|
|||||||
*result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
|
*result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
int64 actual_payload = arg.template Get<int64>();
|
int64_t actual_payload = arg.template Get<int64_t>();
|
||||||
if (actual_payload != payload) {
|
if (actual_payload != payload) {
|
||||||
*result_listener << "with incorrect payload = " << actual_payload;
|
*result_listener << "with incorrect payload = " << actual_payload;
|
||||||
return false;
|
return false;
|
||||||
@@ -137,18 +137,18 @@ class ReproducibleJitterWithReflectionStrategyForTesting
|
|||||||
//
|
//
|
||||||
// An EXPECT will fail if sequence is less than the number requested during
|
// An EXPECT will fail if sequence is less than the number requested during
|
||||||
// processing.
|
// processing.
|
||||||
static std::vector<uint64> random_sequence;
|
static std::vector<uint64_t> random_sequence;
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
virtual uint64 GetNextRandom(uint64 n) {
|
virtual uint64_t GetNextRandom(uint64_t n) {
|
||||||
EXPECT_LT(sequence_index_, random_sequence.size());
|
EXPECT_LT(sequence_index_, random_sequence.size());
|
||||||
return random_sequence[sequence_index_++] % n;
|
return random_sequence[sequence_index_++] % n;
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
int32 sequence_index_ = 0;
|
int32_t sequence_index_ = 0;
|
||||||
};
|
};
|
||||||
std::vector<uint64>
|
std::vector<uint64_t>
|
||||||
ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
|
ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
|
||||||
|
|
||||||
// PacketResamplerCalculator child class which injects a specified stream
|
// PacketResamplerCalculator child class which injects a specified stream
|
||||||
@@ -469,7 +469,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
|||||||
}
|
}
|
||||||
)pb"));
|
)pb"));
|
||||||
|
|
||||||
for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
|
for (const int64_t ts : {0, 5000, 10010, 15001, 19990}) {
|
||||||
runner.MutableInputs()->Tag(kDataTag).packets.push_back(
|
runner.MutableInputs()->Tag(kDataTag).packets.push_back(
|
||||||
Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts)));
|
Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts)));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
#include <cmath> // for ceil
|
#include <cmath> // for ceil
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_context.h"
|
#include "mediapipe/framework/calculator_context.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
@@ -97,7 +98,7 @@ class PacketThinnerCalculator : public CalculatorBase {
|
|||||||
cc->Inputs().Index(0).SetAny();
|
cc->Inputs().Index(0).SetAny();
|
||||||
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
|
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
|
||||||
if (cc->InputSidePackets().HasTag(kPeriodTag)) {
|
if (cc->InputSidePackets().HasTag(kPeriodTag)) {
|
||||||
cc->InputSidePackets().Tag(kPeriodTag).Set<int64>();
|
cc->InputSidePackets().Tag(kPeriodTag).Set<int64_t>();
|
||||||
}
|
}
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
@@ -160,8 +161,8 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
|
|||||||
|
|
||||||
thinner_type_ = options.thinner_type();
|
thinner_type_ = options.thinner_type();
|
||||||
// This check enables us to assume only two thinner types exist in Process()
|
// This check enables us to assume only two thinner types exist in Process()
|
||||||
CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
|
ABSL_CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
|
||||||
thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
|
thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
|
||||||
<< "Unsupported thinner type.";
|
<< "Unsupported thinner type.";
|
||||||
|
|
||||||
if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
|
if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
|
||||||
@@ -173,11 +174,12 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
|
|||||||
|
|
||||||
if (cc->InputSidePackets().HasTag(kPeriodTag)) {
|
if (cc->InputSidePackets().HasTag(kPeriodTag)) {
|
||||||
period_ =
|
period_ =
|
||||||
TimestampDiff(cc->InputSidePackets().Tag(kPeriodTag).Get<int64>());
|
TimestampDiff(cc->InputSidePackets().Tag(kPeriodTag).Get<int64_t>());
|
||||||
} else {
|
} else {
|
||||||
period_ = TimestampDiff(options.period());
|
period_ = TimestampDiff(options.period());
|
||||||
}
|
}
|
||||||
CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive.";
|
ABSL_CHECK_LT(TimestampDiff(0), period_)
|
||||||
|
<< "Specified period must be positive.";
|
||||||
|
|
||||||
if (options.has_start_time()) {
|
if (options.has_start_time()) {
|
||||||
start_time_ = Timestamp(options.start_time());
|
start_time_ = Timestamp(options.start_time());
|
||||||
@@ -189,7 +191,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
|
|||||||
|
|
||||||
end_time_ =
|
end_time_ =
|
||||||
options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
|
options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
|
||||||
CHECK_LT(start_time_, end_time_)
|
ABSL_CHECK_LT(start_time_, end_time_)
|
||||||
<< "Invalid PacketThinner: start_time must be earlier than end_time";
|
<< "Invalid PacketThinner: start_time must be earlier than end_time";
|
||||||
|
|
||||||
sync_output_timestamps_ = options.sync_output_timestamps();
|
sync_output_timestamps_ = options.sync_output_timestamps();
|
||||||
@@ -232,7 +234,7 @@ absl::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
|
|||||||
// Emit any saved packets before quitting.
|
// Emit any saved packets before quitting.
|
||||||
if (!saved_packet_.IsEmpty()) {
|
if (!saved_packet_.IsEmpty()) {
|
||||||
// Only sync thinner should have saved packets.
|
// Only sync thinner should have saved packets.
|
||||||
CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
|
ABSL_CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
|
||||||
if (sync_output_timestamps_) {
|
if (sync_output_timestamps_) {
|
||||||
cc->Outputs().Index(0).AddPacket(
|
cc->Outputs().Index(0).AddPacket(
|
||||||
saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
|
saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
|
||||||
@@ -269,7 +271,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
|
|||||||
const Timestamp saved_sync = NearestSyncTimestamp(saved);
|
const Timestamp saved_sync = NearestSyncTimestamp(saved);
|
||||||
const Timestamp now = cc->InputTimestamp();
|
const Timestamp now = cc->InputTimestamp();
|
||||||
const Timestamp now_sync = NearestSyncTimestamp(now);
|
const Timestamp now_sync = NearestSyncTimestamp(now);
|
||||||
CHECK_LE(saved_sync, now_sync);
|
ABSL_CHECK_LE(saved_sync, now_sync);
|
||||||
if (saved_sync == now_sync) {
|
if (saved_sync == now_sync) {
|
||||||
// Saved Packet is in same interval as current packet.
|
// Saved Packet is in same interval as current packet.
|
||||||
// Replace saved packet with current if it is at least as
|
// Replace saved packet with current if it is at least as
|
||||||
@@ -295,20 +297,20 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
|
|||||||
}
|
}
|
||||||
|
|
||||||
Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
|
Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
|
||||||
CHECK_NE(start_time_, Timestamp::Unset())
|
ABSL_CHECK_NE(start_time_, Timestamp::Unset())
|
||||||
<< "Method only valid for sync thinner calculator.";
|
<< "Method only valid for sync thinner calculator.";
|
||||||
|
|
||||||
// Computation is done using int64 arithmetic. No easy way to avoid
|
// Computation is done using int64 arithmetic. No easy way to avoid
|
||||||
// since Timestamps don't support div and multiply.
|
// since Timestamps don't support div and multiply.
|
||||||
const int64 now64 = now.Value();
|
const int64_t now64 = now.Value();
|
||||||
const int64 start64 = start_time_.Value();
|
const int64_t start64 = start_time_.Value();
|
||||||
const int64 period64 = period_.Value();
|
const int64_t period64 = period_.Value();
|
||||||
CHECK_LE(0, period64);
|
ABSL_CHECK_LE(0, period64);
|
||||||
|
|
||||||
// Round now64 to its closest interval (units of period64).
|
// Round now64 to its closest interval (units of period64).
|
||||||
int64 sync64 =
|
int64_t sync64 =
|
||||||
(now64 - start64 + period64 / 2) / period64 * period64 + start64;
|
(now64 - start64 + period64 / 2) / period64 * period64 + start64;
|
||||||
CHECK_LE(abs(now64 - sync64), period64 / 2)
|
ABSL_CHECK_LE(abs(now64 - sync64), period64 / 2)
|
||||||
<< "start64: " << start64 << "; now64: " << now64
|
<< "start64: " << start64 << "; now64: " << now64
|
||||||
<< "; sync64: " << sync64;
|
<< "; sync64: " << sync64;
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,7 @@
|
|||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "absl/strings/str_cat.h"
|
#include "absl/strings/str_cat.h"
|
||||||
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
@@ -70,7 +71,7 @@ class SimpleRunner : public CalculatorRunner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
double GetFrameRate() const {
|
double GetFrameRate() const {
|
||||||
CHECK(!Outputs().Index(0).header.IsEmpty());
|
ABSL_CHECK(!Outputs().Index(0).header.IsEmpty());
|
||||||
return Outputs().Index(0).header.Get<VideoHeader>().frame_rate;
|
return Outputs().Index(0).header.Get<VideoHeader>().frame_rate;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -123,7 +123,10 @@ class PreviousLoopbackCalculator : public Node {
|
|||||||
// However, LOOP packet is empty.
|
// However, LOOP packet is empty.
|
||||||
kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
|
kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
|
||||||
} else {
|
} else {
|
||||||
kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
|
// Avoids sending leftovers to a stream that's already closed.
|
||||||
|
if (!kPrevLoop(cc).IsClosed()) {
|
||||||
|
kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
loop_packets_.pop_front();
|
loop_packets_.pop_front();
|
||||||
main_packet_specs_.pop_front();
|
main_packet_specs_.pop_front();
|
||||||
|
|||||||
@@ -43,8 +43,8 @@ constexpr char kDisallowTag[] = "DISALLOW";
|
|||||||
|
|
||||||
// Returns the timestamp values for a vector of Packets.
|
// Returns the timestamp values for a vector of Packets.
|
||||||
// TODO: puth this kind of test util in a common place.
|
// TODO: puth this kind of test util in a common place.
|
||||||
std::vector<int64> TimestampValues(const std::vector<Packet>& packets) {
|
std::vector<int64_t> TimestampValues(const std::vector<Packet>& packets) {
|
||||||
std::vector<int64> result;
|
std::vector<int64_t> result;
|
||||||
for (const Packet& packet : packets) {
|
for (const Packet& packet : packets) {
|
||||||
result.push_back(packet.Timestamp().Value());
|
result.push_back(packet.Timestamp().Value());
|
||||||
}
|
}
|
||||||
@@ -371,7 +371,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
|
|||||||
for (int main_ts = 0; main_ts < 50; ++main_ts) {
|
for (int main_ts = 0; main_ts < 50; ++main_ts) {
|
||||||
send_packet("in", main_ts);
|
send_packet("in", main_ts);
|
||||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||||
std::vector<int64> ts_values = TimestampValues(outputs);
|
std::vector<int64_t> ts_values = TimestampValues(outputs);
|
||||||
EXPECT_EQ(ts_values.size(), main_ts + 1);
|
EXPECT_EQ(ts_values.size(), main_ts + 1);
|
||||||
for (int j = 0; j < main_ts + 1; ++j) {
|
for (int j = 0; j < main_ts + 1; ++j) {
|
||||||
EXPECT_EQ(ts_values[j], j);
|
EXPECT_EQ(ts_values[j], j);
|
||||||
|
|||||||
@@ -14,6 +14,7 @@
|
|||||||
|
|
||||||
#include <deque>
|
#include <deque>
|
||||||
|
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h"
|
#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h"
|
||||||
#include "mediapipe/framework/api2/node.h"
|
#include "mediapipe/framework/api2/node.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
@@ -101,7 +102,7 @@ void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
|
|||||||
kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
|
kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
|
||||||
packet_cache_.pop_front();
|
packet_cache_.pop_front();
|
||||||
} else if (emit_empty_packets_before_first_packet_) {
|
} else if (emit_empty_packets_before_first_packet_) {
|
||||||
LOG(FATAL) << "Not supported yet";
|
ABSL_LOG(FATAL) << "Not supported yet";
|
||||||
}
|
}
|
||||||
// Store current packet for later output.
|
// Store current packet for later output.
|
||||||
packet_cache_.push_back(kIn(cc).packet());
|
packet_cache_.push_back(kIn(cc).packet());
|
||||||
|
|||||||
@@ -121,7 +121,7 @@ absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
|
|||||||
if (cc->Outputs().HasTag(kTagAtTimestamp)) {
|
if (cc->Outputs().HasTag(kTagAtTimestamp)) {
|
||||||
RET_CHECK_EQ(num_entries + 1, cc->InputSidePackets().NumEntries())
|
RET_CHECK_EQ(num_entries + 1, cc->InputSidePackets().NumEntries())
|
||||||
<< "For AT_TIMESTAMP tag, 2 input side packets are required.";
|
<< "For AT_TIMESTAMP tag, 2 input side packets are required.";
|
||||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set<int64>();
|
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set<int64_t>();
|
||||||
} else {
|
} else {
|
||||||
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
|
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
|
||||||
<< "Same number of input side packets and output streams is required.";
|
<< "Same number of input side packets and output streams is required.";
|
||||||
@@ -178,8 +178,8 @@ absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
|
|||||||
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
||||||
}
|
}
|
||||||
} else if (cc->Outputs().HasTag(kTagAtTimestamp)) {
|
} else if (cc->Outputs().HasTag(kTagAtTimestamp)) {
|
||||||
int64 timestamp =
|
int64_t timestamp =
|
||||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get<int64>();
|
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get<int64_t>();
|
||||||
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
||||||
cc->Outputs()
|
cc->Outputs()
|
||||||
.Get(output_tag_, i)
|
.Get(output_tag_, i)
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
|
|
||||||
#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
|
#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/formats/body_rig.pb.h"
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
#include "mediapipe/framework/port/canonical_errors.h"
|
#include "mediapipe/framework/port/canonical_errors.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
@@ -196,6 +197,18 @@ class SplitLandmarkListCalculator
|
|||||||
};
|
};
|
||||||
REGISTER_CALCULATOR(SplitLandmarkListCalculator);
|
REGISTER_CALCULATOR(SplitLandmarkListCalculator);
|
||||||
|
|
||||||
|
class SplitJointListCalculator : public SplitListsCalculator<Joint, JointList> {
|
||||||
|
protected:
|
||||||
|
int ListSize(const JointList& list) const override {
|
||||||
|
return list.joint_size();
|
||||||
|
}
|
||||||
|
const Joint GetItem(const JointList& list, int idx) const override {
|
||||||
|
return list.joint(idx);
|
||||||
|
}
|
||||||
|
Joint* AddItem(JointList& list) const override { return list.add_joint(); }
|
||||||
|
};
|
||||||
|
REGISTER_CALCULATOR(SplitJointListCalculator);
|
||||||
|
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|
||||||
// NOLINTNEXTLINE
|
// NOLINTNEXTLINE
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
|
|
||||||
#include "mediapipe/framework/formats/classification.pb.h"
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
#include "mediapipe/framework/formats/detection.pb.h"
|
#include "mediapipe/framework/formats/detection.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image.h"
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
#include "mediapipe/framework/formats/rect.pb.h"
|
#include "mediapipe/framework/formats/rect.pb.h"
|
||||||
@@ -86,4 +87,12 @@ REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
|
|||||||
typedef SplitVectorCalculator<float, false> SplitFloatVectorCalculator;
|
typedef SplitVectorCalculator<float, false> SplitFloatVectorCalculator;
|
||||||
REGISTER_CALCULATOR(SplitFloatVectorCalculator);
|
REGISTER_CALCULATOR(SplitFloatVectorCalculator);
|
||||||
|
|
||||||
|
typedef SplitVectorCalculator<mediapipe::Image, false>
|
||||||
|
SplitImageVectorCalculator;
|
||||||
|
REGISTER_CALCULATOR(SplitImageVectorCalculator);
|
||||||
|
|
||||||
|
typedef SplitVectorCalculator<std::array<float, 16>, false>
|
||||||
|
SplitAffineMatrixVectorCalculator;
|
||||||
|
REGISTER_CALCULATOR(SplitAffineMatrixVectorCalculator);
|
||||||
|
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|||||||
@@ -12,11 +12,13 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "mediapipe/framework/api2/node.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
#include "mediapipe/framework/timestamp.h"
|
#include "mediapipe/framework/timestamp.h"
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
|
namespace api2 {
|
||||||
|
|
||||||
// A calculator that takes a packet of an input stream and converts it to an
|
// A calculator that takes a packet of an input stream and converts it to an
|
||||||
// output side packet. This calculator only works under the assumption that the
|
// output side packet. This calculator only works under the assumption that the
|
||||||
@@ -28,21 +30,21 @@ namespace mediapipe {
|
|||||||
// input_stream: "stream"
|
// input_stream: "stream"
|
||||||
// output_side_packet: "side_packet"
|
// output_side_packet: "side_packet"
|
||||||
// }
|
// }
|
||||||
class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
|
class StreamToSidePacketCalculator : public Node {
|
||||||
public:
|
public:
|
||||||
static absl::Status GetContract(mediapipe::CalculatorContract* cc) {
|
static constexpr Input<AnyType>::Optional kIn{""};
|
||||||
cc->Inputs().Index(0).SetAny();
|
static constexpr SideOutput<SameType<kIn>> kOut{""};
|
||||||
cc->OutputSidePackets().Index(0).SetAny();
|
|
||||||
return absl::OkStatus();
|
MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
|
||||||
}
|
|
||||||
|
|
||||||
absl::Status Process(mediapipe::CalculatorContext* cc) override {
|
absl::Status Process(mediapipe::CalculatorContext* cc) override {
|
||||||
mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
|
kOut(cc).Set(
|
||||||
cc->OutputSidePackets().Index(0).Set(
|
kIn(cc).packet().As<AnyType>().At(mediapipe::Timestamp::Unset()));
|
||||||
packet.At(mediapipe::Timestamp::Unset()));
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
REGISTER_CALCULATOR(StreamToSidePacketCalculator);
|
|
||||||
|
|
||||||
|
MEDIAPIPE_REGISTER_NODE(StreamToSidePacketCalculator);
|
||||||
|
|
||||||
|
} // namespace api2
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|||||||
@@ -64,16 +64,16 @@ REGISTER_CALCULATOR(StringToIntCalculator);
|
|||||||
using StringToUintCalculator = StringToIntCalculatorTemplate<unsigned int>;
|
using StringToUintCalculator = StringToIntCalculatorTemplate<unsigned int>;
|
||||||
REGISTER_CALCULATOR(StringToUintCalculator);
|
REGISTER_CALCULATOR(StringToUintCalculator);
|
||||||
|
|
||||||
using StringToInt32Calculator = StringToIntCalculatorTemplate<int32>;
|
using StringToInt32Calculator = StringToIntCalculatorTemplate<int32_t>;
|
||||||
REGISTER_CALCULATOR(StringToInt32Calculator);
|
REGISTER_CALCULATOR(StringToInt32Calculator);
|
||||||
|
|
||||||
using StringToUint32Calculator = StringToIntCalculatorTemplate<uint32>;
|
using StringToUint32Calculator = StringToIntCalculatorTemplate<uint32_t>;
|
||||||
REGISTER_CALCULATOR(StringToUint32Calculator);
|
REGISTER_CALCULATOR(StringToUint32Calculator);
|
||||||
|
|
||||||
using StringToInt64Calculator = StringToIntCalculatorTemplate<int64>;
|
using StringToInt64Calculator = StringToIntCalculatorTemplate<int64_t>;
|
||||||
REGISTER_CALCULATOR(StringToInt64Calculator);
|
REGISTER_CALCULATOR(StringToInt64Calculator);
|
||||||
|
|
||||||
using StringToUint64Calculator = StringToIntCalculatorTemplate<uint64>;
|
using StringToUint64Calculator = StringToIntCalculatorTemplate<uint64_t>;
|
||||||
REGISTER_CALCULATOR(StringToUint64Calculator);
|
REGISTER_CALCULATOR(StringToUint64Calculator);
|
||||||
|
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/port/status.h"
|
||||||
|
|
||||||
|
namespace mediapipe {
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
constexpr char kInputValueTag[] = "IN";
|
||||||
|
constexpr char kTickerTag[] = "TICK";
|
||||||
|
constexpr char kOutputTag[] = "OUT";
|
||||||
|
constexpr char kIndicationTag[] = "FLAG";
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
// For every packet received on the TICK stream, if the IN stream is not
|
||||||
|
// empty - emit its value as is as OUT. Otherwise output a default packet.
|
||||||
|
// FLAG outputs true every time the default value has been used. It does not
|
||||||
|
// output anything when IN has a value.
|
||||||
|
//
|
||||||
|
// Example config:
|
||||||
|
// node {
|
||||||
|
// calculator: "ValueOrDefaultCalculator"
|
||||||
|
// input_stream: "IN:sometimes_missing_value"
|
||||||
|
// input_stream: "TICK:clock"
|
||||||
|
// output_stream: "OUT:value_or_default"
|
||||||
|
// output_stream: "FLAG:used_default"
|
||||||
|
// input_side_packet: "default"
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// TODO: Consider adding an option for a default value as a input-stream
|
||||||
|
// instead of a side-packet, so it will enable using standard calculators
|
||||||
|
// instead of creating a new packet-generators. It will also allow a dynamic
|
||||||
|
// default value.
|
||||||
|
class ValueOrDefaultCalculator : public mediapipe::CalculatorBase {
|
||||||
|
public:
|
||||||
|
ValueOrDefaultCalculator() {}
|
||||||
|
|
||||||
|
ValueOrDefaultCalculator(const ValueOrDefaultCalculator&) = delete;
|
||||||
|
ValueOrDefaultCalculator& operator=(const ValueOrDefaultCalculator&) = delete;
|
||||||
|
|
||||||
|
static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc) {
|
||||||
|
cc->Inputs().Tag(kInputValueTag).SetAny();
|
||||||
|
cc->Inputs().Tag(kTickerTag).SetAny();
|
||||||
|
cc->Outputs().Tag(kOutputTag).SetSameAs(&cc->Inputs().Tag(kInputValueTag));
|
||||||
|
cc->Outputs().Tag(kIndicationTag).Set<bool>();
|
||||||
|
cc->InputSidePackets().Index(0).SetSameAs(
|
||||||
|
&cc->Inputs().Tag(kInputValueTag));
|
||||||
|
|
||||||
|
return mediapipe::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
mediapipe::Status Open(mediapipe::CalculatorContext* cc) override {
|
||||||
|
if (!cc->Inputs().Tag(kInputValueTag).Header().IsEmpty()) {
|
||||||
|
cc->Outputs()
|
||||||
|
.Tag(kOutputTag)
|
||||||
|
.SetHeader(cc->Inputs().Tag(kInputValueTag).Header());
|
||||||
|
}
|
||||||
|
default_ = cc->InputSidePackets().Index(0);
|
||||||
|
cc->SetOffset(mediapipe::TimestampDiff(0));
|
||||||
|
return mediapipe::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
mediapipe::Status Process(mediapipe::CalculatorContext* cc) override {
|
||||||
|
// Output according to the TICK signal.
|
||||||
|
if (cc->Inputs().Tag(kTickerTag).IsEmpty()) {
|
||||||
|
return mediapipe::OkStatus();
|
||||||
|
}
|
||||||
|
if (!cc->Inputs().Tag(kInputValueTag).IsEmpty()) {
|
||||||
|
// Output the input as is:
|
||||||
|
cc->Outputs()
|
||||||
|
.Tag(kOutputTag)
|
||||||
|
.AddPacket(cc->Inputs().Tag(kInputValueTag).Value());
|
||||||
|
} else {
|
||||||
|
// Output default:
|
||||||
|
cc->Outputs()
|
||||||
|
.Tag(kOutputTag)
|
||||||
|
.AddPacket(default_.At(cc->InputTimestamp()));
|
||||||
|
cc->Outputs()
|
||||||
|
.Tag(kIndicationTag)
|
||||||
|
.Add(new bool(true), cc->InputTimestamp());
|
||||||
|
}
|
||||||
|
return mediapipe::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
// The default value to replicate every time there is no new value.
|
||||||
|
mediapipe::Packet default_;
|
||||||
|
};
|
||||||
|
|
||||||
|
REGISTER_CALCULATOR(ValueOrDefaultCalculator);
|
||||||
|
|
||||||
|
} // namespace mediapipe
|
||||||
@@ -0,0 +1,240 @@
|
|||||||
|
#include <algorithm>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/calculator_runner.h"
|
||||||
|
#include "mediapipe/framework/packet.h"
|
||||||
|
#include "mediapipe/framework/port/gmock.h"
|
||||||
|
#include "mediapipe/framework/port/gtest.h"
|
||||||
|
#include "mediapipe/framework/port/status_matchers.h"
|
||||||
|
|
||||||
|
namespace mediapipe {
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
using ::testing::AllOf;
|
||||||
|
using ::testing::ContainerEq;
|
||||||
|
using ::testing::Each;
|
||||||
|
using ::testing::ElementsAre;
|
||||||
|
using ::testing::IsEmpty;
|
||||||
|
using ::testing::SizeIs;
|
||||||
|
using ::testing::Test;
|
||||||
|
|
||||||
|
const int kDefaultValue = 0;
|
||||||
|
|
||||||
|
// Utility to a create a mediapipe graph runner with the tested calculator and a
|
||||||
|
// default value, for all the tests.
|
||||||
|
class ValueOrDefaultRunner : public mediapipe::CalculatorRunner {
|
||||||
|
public:
|
||||||
|
ValueOrDefaultRunner()
|
||||||
|
: mediapipe::CalculatorRunner(R"pb(
|
||||||
|
calculator: "ValueOrDefaultCalculator"
|
||||||
|
input_stream: "IN:in"
|
||||||
|
input_stream: "TICK:tick"
|
||||||
|
input_side_packet: "default"
|
||||||
|
output_stream: "OUT:out"
|
||||||
|
output_stream: "FLAG:used_default"
|
||||||
|
)pb") {
|
||||||
|
MutableSidePackets()->Index(0) = mediapipe::MakePacket<int>(kDefaultValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Utility to push inputs to the runner to the TICK stream, so we could easily
|
||||||
|
// tick.
|
||||||
|
void TickAt(int64_t time) {
|
||||||
|
// The type or value of the stream isn't relevant, we use just a bool.
|
||||||
|
MutableInputs()->Tag("TICK").packets.push_back(
|
||||||
|
mediapipe::Adopt(new bool(false)).At(mediapipe::Timestamp(time)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Utility to push the real inputs to the runner (IN stream).
|
||||||
|
void ProvideInput(int64_t time, int value) {
|
||||||
|
MutableInputs()->Tag("IN").packets.push_back(
|
||||||
|
mediapipe::Adopt(new int(value)).At(mediapipe::Timestamp(time)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extracts the timestamps (as int64) of the output stream of the calculator.
|
||||||
|
std::vector<int64_t> GetOutputTimestamps() const {
|
||||||
|
std::vector<int64_t> timestamps;
|
||||||
|
for (const mediapipe::Packet& packet : Outputs().Tag("OUT").packets) {
|
||||||
|
timestamps.emplace_back(packet.Timestamp().Value());
|
||||||
|
}
|
||||||
|
return timestamps;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extracts the values from the output stream of the calculator.
|
||||||
|
std::vector<int> GetOutputValues() const {
|
||||||
|
std::vector<int> values;
|
||||||
|
for (const mediapipe::Packet& packet : Outputs().Tag("OUT").packets) {
|
||||||
|
values.emplace_back(packet.Get<int>());
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extracts the timestamps (as int64) of the flag stream, which indicates on
|
||||||
|
// times without an input value (i.e. using the default value).
|
||||||
|
std::vector<int64_t> GetFlagTimestamps() const {
|
||||||
|
std::vector<int64_t> timestamps;
|
||||||
|
for (const mediapipe::Packet& packet : Outputs().Tag("FLAG").packets) {
|
||||||
|
timestamps.emplace_back(packet.Timestamp().Value());
|
||||||
|
}
|
||||||
|
return timestamps;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extracts the output from the flags stream (which should always be true).
|
||||||
|
std::vector<bool> GetFlagValues() const {
|
||||||
|
std::vector<bool> flags;
|
||||||
|
for (const mediapipe::Packet& packet : Outputs().Tag("FLAG").packets) {
|
||||||
|
flags.emplace_back(packet.Get<bool>());
|
||||||
|
}
|
||||||
|
return flags;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// To be used as input values:
|
||||||
|
std::vector<int> GetIntegersRange(int size) {
|
||||||
|
std::vector<int> result;
|
||||||
|
for (int i = 0; i < size; ++i) {
|
||||||
|
// We start with default-value+1 so it won't contain the default value.
|
||||||
|
result.push_back(kDefaultValue + 1 + i);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ValueOrDefaultCalculatorTest, NoInputs) {
|
||||||
|
// Check that when no real inputs are provided - we get the default value over
|
||||||
|
// and over, with the correct timestamps.
|
||||||
|
ValueOrDefaultRunner runner;
|
||||||
|
const std::vector<int64_t> ticks = {0, 1, 2, 5, 8, 12, 33, 231};
|
||||||
|
|
||||||
|
for (int tick : ticks) {
|
||||||
|
runner.TickAt(tick);
|
||||||
|
}
|
||||||
|
|
||||||
|
MP_EXPECT_OK(runner.Run());
|
||||||
|
|
||||||
|
// Make sure we get the right timestamps:
|
||||||
|
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
||||||
|
// All should be default value:
|
||||||
|
EXPECT_THAT(runner.GetOutputValues(),
|
||||||
|
AllOf(Each(kDefaultValue), SizeIs(ticks.size())));
|
||||||
|
// We should get the default indication all the time:
|
||||||
|
EXPECT_THAT(runner.GetFlagTimestamps(), ContainerEq(ticks));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ValueOrDefaultCalculatorTest, NeverDefault) {
|
||||||
|
// Check that when we provide the inputs on time - we get them as outputs.
|
||||||
|
ValueOrDefaultRunner runner;
|
||||||
|
const std::vector<int64_t> ticks = {0, 1, 2, 5, 8, 12, 33, 231};
|
||||||
|
const std::vector<int> values = GetIntegersRange(ticks.size());
|
||||||
|
|
||||||
|
for (int i = 0; i < ticks.size(); ++i) {
|
||||||
|
runner.TickAt(ticks[i]);
|
||||||
|
runner.ProvideInput(ticks[i], values[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
MP_EXPECT_OK(runner.Run());
|
||||||
|
|
||||||
|
// Make sure we get the right timestamps:
|
||||||
|
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
||||||
|
// Should get the inputs values:
|
||||||
|
EXPECT_THAT(runner.GetOutputValues(), ContainerEq(values));
|
||||||
|
// We should never get the default indication:
|
||||||
|
EXPECT_THAT(runner.GetFlagTimestamps(), IsEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ValueOrDefaultCalculatorTest, DefaultAndValues) {
|
||||||
|
// Check that when we provide inputs only part of the time - we get them, but
|
||||||
|
// defaults at the missing times.
|
||||||
|
// That's the usual use case for this calculator.
|
||||||
|
ValueOrDefaultRunner runner;
|
||||||
|
const std::vector<int64_t> ticks = {0, 1, 5, 8, 12, 231};
|
||||||
|
// Provide inputs only part of the ticks.
|
||||||
|
// Chosen so there will be defaults before the first input, between the
|
||||||
|
// inputs and after the last input.
|
||||||
|
const std::vector<int64_t> in_ticks = {/*0,*/ 1, 5, /*8,*/ 12, /*, 231*/};
|
||||||
|
const std::vector<int> in_values = GetIntegersRange(in_ticks.size());
|
||||||
|
|
||||||
|
for (int tick : ticks) {
|
||||||
|
runner.TickAt(tick);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < in_ticks.size(); ++i) {
|
||||||
|
runner.ProvideInput(in_ticks[i], in_values[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
MP_EXPECT_OK(runner.Run());
|
||||||
|
|
||||||
|
// Make sure we get all the timestamps:
|
||||||
|
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
||||||
|
// The timestamps of the flag should be exactly the ones not in in_ticks.
|
||||||
|
EXPECT_THAT(runner.GetFlagTimestamps(), ElementsAre(0, 8, 231));
|
||||||
|
// And the values are default in these times, and the input values for
|
||||||
|
// in_ticks.
|
||||||
|
EXPECT_THAT(
|
||||||
|
runner.GetOutputValues(),
|
||||||
|
ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ValueOrDefaultCalculatorTest, TimestampsMissmatch) {
|
||||||
|
// Check that when we provide the inputs not on time - we don't get them.
|
||||||
|
ValueOrDefaultRunner runner;
|
||||||
|
const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};
|
||||||
|
// The timestamps chosen so it will be before the first tick, in between ticks
|
||||||
|
// and after the last one. Also - more inputs than ticks.
|
||||||
|
const std::vector<int64_t> in_ticks = {0, 3, 4, 6, 7, 9, 10,
|
||||||
|
11, 13, 14, 15, 16, 232};
|
||||||
|
const std::vector<int> in_values = GetIntegersRange(in_ticks.size());
|
||||||
|
for (int tick : ticks) {
|
||||||
|
runner.TickAt(tick);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < in_ticks.size(); ++i) {
|
||||||
|
runner.ProvideInput(in_ticks[i], in_values[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
MP_EXPECT_OK(runner.Run());
|
||||||
|
|
||||||
|
// Non of the in_ticks should be inserted:
|
||||||
|
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
||||||
|
EXPECT_THAT(runner.GetOutputValues(),
|
||||||
|
AllOf(Each(kDefaultValue), SizeIs(ticks.size())));
|
||||||
|
// All (and only) ticks should get the default.
|
||||||
|
EXPECT_THAT(runner.GetFlagTimestamps(), ContainerEq(ticks));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ValueOrDefaultCalculatorTest, FlagValue) {
|
||||||
|
// Since we anyway suppose that the Flag is a bool - there is nothing
|
||||||
|
// interesting to check, but we should check once that the value is the right
|
||||||
|
// (true) one.
|
||||||
|
ValueOrDefaultRunner runner;
|
||||||
|
runner.TickAt(0);
|
||||||
|
MP_EXPECT_OK(runner.Run());
|
||||||
|
EXPECT_THAT(runner.GetFlagValues(), ElementsAre(true));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ValueOrDefaultCalculatorTest, FullTest) {
|
||||||
|
// Make sure that nothing gets wrong with an input that have both right and
|
||||||
|
// wrong timestamps, some defaults etc.
|
||||||
|
ValueOrDefaultRunner runner;
|
||||||
|
const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};
|
||||||
|
const std::vector<int64_t> in_ticks = {0, 2, 4, 6, 8, 9, 12, 33, 54, 232};
|
||||||
|
const std::vector<int> in_values = GetIntegersRange(in_ticks.size());
|
||||||
|
|
||||||
|
for (int tick : ticks) {
|
||||||
|
runner.TickAt(tick);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < in_ticks.size(); ++i) {
|
||||||
|
runner.ProvideInput(in_ticks[i], in_values[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
MP_EXPECT_OK(runner.Run());
|
||||||
|
|
||||||
|
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
||||||
|
// Calculated by hand:
|
||||||
|
EXPECT_THAT(
|
||||||
|
runner.GetOutputValues(),
|
||||||
|
ElementsAre(kDefaultValue, 2, kDefaultValue, 5, 7, 8, kDefaultValue));
|
||||||
|
EXPECT_THAT(runner.GetFlagTimestamps(), ElementsAre(1, 5, 231));
|
||||||
|
EXPECT_THAT(runner.GetFlagValues(), AllOf(Each(true), SizeIs(3)));
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
} // namespace mediapipe
|
||||||
@@ -97,6 +97,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:source_location",
|
"//mediapipe/framework/port:source_location",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
@@ -125,6 +126,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
@@ -135,7 +137,6 @@ cc_library(
|
|||||||
deps = [
|
deps = [
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/formats:image_frame_opencv",
|
"//mediapipe/framework/formats:image_frame_opencv",
|
||||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
],
|
],
|
||||||
@@ -152,11 +153,11 @@ cc_library(
|
|||||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||||
"//mediapipe/framework/formats:image_frame",
|
"//mediapipe/framework/formats:image_frame",
|
||||||
"//mediapipe/framework/formats:image_frame_opencv",
|
"//mediapipe/framework/formats:image_frame_opencv",
|
||||||
"//mediapipe/framework/port:logging",
|
|
||||||
"//mediapipe/framework/port:opencv_core",
|
"//mediapipe/framework/port:opencv_core",
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/port:vector",
|
"//mediapipe/framework/port:vector",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
@@ -203,6 +204,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/port:vector",
|
"//mediapipe/framework/port:vector",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
@@ -262,9 +264,12 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/gpu:scale_mode_cc_proto",
|
"//mediapipe/gpu:scale_mode_cc_proto",
|
||||||
|
"@com_google_absl//absl/status",
|
||||||
|
"@com_google_absl//absl/strings",
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
|
"//mediapipe/gpu:gl_base_hdr",
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
"//mediapipe/gpu:gl_calculator_helper",
|
||||||
"//mediapipe/gpu:gl_quad_renderer",
|
"//mediapipe/gpu:gl_quad_renderer",
|
||||||
"//mediapipe/gpu:gl_simple_shaders",
|
"//mediapipe/gpu:gl_simple_shaders",
|
||||||
@@ -274,6 +279,36 @@ cc_library(
|
|||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cc_test(
|
||||||
|
name = "image_transformation_calculator_test",
|
||||||
|
srcs = ["image_transformation_calculator_test.cc"],
|
||||||
|
data = ["//mediapipe/calculators/image/testdata:test_images"],
|
||||||
|
tags = [
|
||||||
|
"desktop_only_test",
|
||||||
|
],
|
||||||
|
deps = [
|
||||||
|
":image_transformation_calculator",
|
||||||
|
"//mediapipe/framework:calculator_cc_proto",
|
||||||
|
"//mediapipe/framework:calculator_framework",
|
||||||
|
"//mediapipe/framework:calculator_runner",
|
||||||
|
"//mediapipe/framework/deps:file_path",
|
||||||
|
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:image_frame",
|
||||||
|
"//mediapipe/framework/formats:image_frame_opencv",
|
||||||
|
"//mediapipe/framework/port:gtest",
|
||||||
|
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||||
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
|
"//mediapipe/framework/port:parse_text_proto",
|
||||||
|
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
|
||||||
|
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
|
||||||
|
"//third_party:opencv",
|
||||||
|
"@com_google_absl//absl/container:flat_hash_set",
|
||||||
|
"@com_google_absl//absl/flags:flag",
|
||||||
|
"@com_google_absl//absl/strings",
|
||||||
|
"@com_google_googletest//:gtest_main",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
cc_library(
|
cc_library(
|
||||||
name = "image_cropping_calculator",
|
name = "image_cropping_calculator",
|
||||||
srcs = ["image_cropping_calculator.cc"],
|
srcs = ["image_cropping_calculator.cc"],
|
||||||
@@ -301,6 +336,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
@@ -317,6 +353,7 @@ cc_library(
|
|||||||
cc_test(
|
cc_test(
|
||||||
name = "image_cropping_calculator_test",
|
name = "image_cropping_calculator_test",
|
||||||
srcs = ["image_cropping_calculator_test.cc"],
|
srcs = ["image_cropping_calculator_test.cc"],
|
||||||
|
tags = ["not_run:arm"],
|
||||||
deps = [
|
deps = [
|
||||||
":image_cropping_calculator",
|
":image_cropping_calculator",
|
||||||
":image_cropping_calculator_cc_proto",
|
":image_cropping_calculator_cc_proto",
|
||||||
@@ -396,6 +433,7 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:logging",
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@@ -420,6 +458,8 @@ cc_library(
|
|||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:image_frame_util",
|
"//mediapipe/util:image_frame_util",
|
||||||
|
"@com_google_absl//absl/log:absl_check",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@libyuv",
|
"@libyuv",
|
||||||
],
|
],
|
||||||
@@ -625,9 +665,9 @@ cc_library(
|
|||||||
"//mediapipe/framework/formats:image",
|
"//mediapipe/framework/formats:image",
|
||||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||||
"//mediapipe/framework/formats:image_frame",
|
"//mediapipe/framework/formats:image_frame",
|
||||||
"//mediapipe/framework/port:logging",
|
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/port:vector",
|
"//mediapipe/framework/port:vector",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
@@ -650,6 +690,7 @@ cc_library(
|
|||||||
cc_test(
|
cc_test(
|
||||||
name = "segmentation_smoothing_calculator_test",
|
name = "segmentation_smoothing_calculator_test",
|
||||||
srcs = ["segmentation_smoothing_calculator_test.cc"],
|
srcs = ["segmentation_smoothing_calculator_test.cc"],
|
||||||
|
tags = ["not_run:arm"],
|
||||||
deps = [
|
deps = [
|
||||||
":image_clone_calculator",
|
":image_clone_calculator",
|
||||||
":image_clone_calculator_cc_proto",
|
":image_clone_calculator_cc_proto",
|
||||||
@@ -664,6 +705,7 @@ cc_test(
|
|||||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
"//mediapipe/framework/port:parse_text_proto",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -686,6 +728,7 @@ cc_library(
|
|||||||
"//mediapipe/gpu:gpu_buffer",
|
"//mediapipe/gpu:gpu_buffer",
|
||||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||||
"//mediapipe/gpu:shader_util",
|
"//mediapipe/gpu:shader_util",
|
||||||
|
"@com_google_absl//absl/log:absl_log",
|
||||||
"@com_google_absl//absl/memory",
|
"@com_google_absl//absl/memory",
|
||||||
"@com_google_absl//absl/status",
|
"@com_google_absl//absl/status",
|
||||||
"@com_google_absl//absl/status:statusor",
|
"@com_google_absl//absl/status:statusor",
|
||||||
@@ -771,7 +814,10 @@ cc_test(
|
|||||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero_interp_cubic.png",
|
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero_interp_cubic.png",
|
||||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
|
"//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
|
||||||
],
|
],
|
||||||
tags = ["desktop_only_test"],
|
tags = [
|
||||||
|
"desktop_only_test",
|
||||||
|
"not_run:arm",
|
||||||
|
],
|
||||||
deps = [
|
deps = [
|
||||||
":affine_transformation",
|
":affine_transformation",
|
||||||
":image_transformation_calculator",
|
":image_transformation_calculator",
|
||||||
|
|||||||
@@ -20,6 +20,7 @@
|
|||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
#include "Eigen/Geometry"
|
#include "Eigen/Geometry"
|
||||||
#include "Eigen/LU"
|
#include "Eigen/LU"
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "absl/memory/memory.h"
|
#include "absl/memory/memory.h"
|
||||||
#include "absl/status/status.h"
|
#include "absl/status/status.h"
|
||||||
#include "absl/status/statusor.h"
|
#include "absl/status/statusor.h"
|
||||||
@@ -53,6 +54,10 @@ bool IsMatrixVerticalFlipNeeded(GpuOrigin::Mode gpu_origin) {
|
|||||||
#endif // __APPLE__
|
#endif // __APPLE__
|
||||||
case GpuOrigin::TOP_LEFT:
|
case GpuOrigin::TOP_LEFT:
|
||||||
return false;
|
return false;
|
||||||
|
default:
|
||||||
|
ABSL_LOG(ERROR) << "Incorrect GpuOrigin: "
|
||||||
|
<< static_cast<int>(gpu_origin);
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -218,7 +223,7 @@ class GlTextureWarpAffineRunner
|
|||||||
absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble,
|
absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble,
|
||||||
interpolation_def, kFragShader);
|
interpolation_def, kFragShader);
|
||||||
|
|
||||||
ASSIGN_OR_RETURN(program_, create_fn(vert_src, frag_src));
|
MP_ASSIGN_OR_RETURN(program_, create_fn(vert_src, frag_src));
|
||||||
|
|
||||||
auto create_custom_zero_fn = [&]() -> absl::StatusOr<Program> {
|
auto create_custom_zero_fn = [&]() -> absl::StatusOr<Program> {
|
||||||
std::string custom_zero_border_mode_def = R"(
|
std::string custom_zero_border_mode_def = R"(
|
||||||
@@ -231,10 +236,10 @@ class GlTextureWarpAffineRunner
|
|||||||
};
|
};
|
||||||
#if GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
#if GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
||||||
if (!IsGlClampToBorderSupported(gl_helper_->GetGlContext())) {
|
if (!IsGlClampToBorderSupported(gl_helper_->GetGlContext())) {
|
||||||
ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
MP_ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
MP_ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
||||||
#endif // GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
#endif // GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
||||||
|
|
||||||
glGenFramebuffers(1, &framebuffer_);
|
glGenFramebuffers(1, &framebuffer_);
|
||||||
@@ -384,6 +389,8 @@ class GlTextureWarpAffineRunner
|
|||||||
glActiveTexture(GL_TEXTURE0);
|
glActiveTexture(GL_TEXTURE0);
|
||||||
glBindTexture(GL_TEXTURE_2D, 0);
|
glBindTexture(GL_TEXTURE_2D, 0);
|
||||||
|
|
||||||
|
glFlush();
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "absl/strings/str_replace.h"
|
#include "absl/strings/str_replace.h"
|
||||||
#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h"
|
#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
@@ -112,7 +113,7 @@ class BilateralFilterCalculator : public CalculatorBase {
|
|||||||
REGISTER_CALCULATOR(BilateralFilterCalculator);
|
REGISTER_CALCULATOR(BilateralFilterCalculator);
|
||||||
|
|
||||||
absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
|
absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
|
||||||
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||||
|
|
||||||
if (cc->Inputs().HasTag(kInputFrameTag) &&
|
if (cc->Inputs().HasTag(kInputFrameTag) &&
|
||||||
cc->Inputs().HasTag(kInputFrameTagGpu)) {
|
cc->Inputs().HasTag(kInputFrameTagGpu)) {
|
||||||
@@ -183,8 +184,8 @@ absl::Status BilateralFilterCalculator::Open(CalculatorContext* cc) {
|
|||||||
|
|
||||||
sigma_color_ = options_.sigma_color();
|
sigma_color_ = options_.sigma_color();
|
||||||
sigma_space_ = options_.sigma_space();
|
sigma_space_ = options_.sigma_space();
|
||||||
CHECK_GE(sigma_color_, 0.0);
|
ABSL_CHECK_GE(sigma_color_, 0.0);
|
||||||
CHECK_GE(sigma_space_, 0.0);
|
ABSL_CHECK_GE(sigma_space_, 0.0);
|
||||||
if (!use_gpu_) sigma_color_ *= 255.0;
|
if (!use_gpu_) sigma_color_ *= 255.0;
|
||||||
|
|
||||||
if (use_gpu_) {
|
if (use_gpu_) {
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/formats/image_frame.h"
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
@@ -25,8 +26,8 @@
|
|||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
namespace {
|
namespace {
|
||||||
void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
|
void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
|
||||||
CHECK(mat->depth() == CV_8U);
|
ABSL_CHECK(mat->depth() == CV_8U);
|
||||||
CHECK(channel < mat->channels());
|
ABSL_CHECK(channel < mat->channels());
|
||||||
const int step = mat->channels();
|
const int step = mat->channels();
|
||||||
for (int r = 0; r < mat->rows; ++r) {
|
for (int r = 0; r < mat->rows; ++r) {
|
||||||
uint8* row_ptr = mat->ptr<uint8>(r);
|
uint8* row_ptr = mat->ptr<uint8>(r);
|
||||||
|
|||||||
@@ -16,6 +16,7 @@
|
|||||||
|
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
|
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "mediapipe/framework/formats/image_frame.h"
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
#include "mediapipe/framework/formats/rect.pb.h"
|
#include "mediapipe/framework/formats/rect.pb.h"
|
||||||
@@ -202,8 +203,9 @@ absl::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
|
|||||||
|
|
||||||
switch (options.border_mode()) {
|
switch (options.border_mode()) {
|
||||||
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
|
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
|
||||||
LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
|
ABSL_LOG(WARNING)
|
||||||
<< "implementation and will fall back into BORDER_REPLICATE";
|
<< "BORDER_ZERO mode is not supported by GPU "
|
||||||
|
<< "implementation and will fall back into BORDER_REPLICATE";
|
||||||
break;
|
break;
|
||||||
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
|
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -92,11 +92,11 @@ absl::StatusOr<ImageFileProperties> GetImageFileProperites(
|
|||||||
properties.set_focal_length_mm(result.FocalLength);
|
properties.set_focal_length_mm(result.FocalLength);
|
||||||
properties.set_focal_length_35mm(result.FocalLengthIn35mm);
|
properties.set_focal_length_35mm(result.FocalLengthIn35mm);
|
||||||
|
|
||||||
ASSIGN_OR_RETURN(auto focal_length_pixels,
|
MP_ASSIGN_OR_RETURN(auto focal_length_pixels,
|
||||||
ComputeFocalLengthInPixels(properties.image_width(),
|
ComputeFocalLengthInPixels(properties.image_width(),
|
||||||
properties.image_height(),
|
properties.image_height(),
|
||||||
properties.focal_length_35mm(),
|
properties.focal_length_35mm(),
|
||||||
properties.focal_length_mm()));
|
properties.focal_length_mm()));
|
||||||
properties.set_focal_length_pixels(focal_length_pixels);
|
properties.set_focal_length_pixels(focal_length_pixels);
|
||||||
|
|
||||||
return properties;
|
return properties;
|
||||||
@@ -151,7 +151,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
|
|||||||
if (cc->InputSidePackets().NumEntries() == 1) {
|
if (cc->InputSidePackets().NumEntries() == 1) {
|
||||||
const std::string& image_bytes =
|
const std::string& image_bytes =
|
||||||
cc->InputSidePackets().Index(0).Get<std::string>();
|
cc->InputSidePackets().Index(0).Get<std::string>();
|
||||||
ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||||
read_properties_ = true;
|
read_properties_ = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,7 +169,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
|
|||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
|
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
|
||||||
ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||||
read_properties_ = true;
|
read_properties_ = true;
|
||||||
}
|
}
|
||||||
if (read_properties_) {
|
if (read_properties_) {
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "absl/status/status.h"
|
||||||
#include "mediapipe/calculators/image/image_transformation_calculator.pb.h"
|
#include "mediapipe/calculators/image/image_transformation_calculator.pb.h"
|
||||||
#include "mediapipe/calculators/image/rotation_mode.pb.h"
|
#include "mediapipe/calculators/image/rotation_mode.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
@@ -27,6 +28,7 @@
|
|||||||
#include "mediapipe/gpu/scale_mode.pb.h"
|
#include "mediapipe/gpu/scale_mode.pb.h"
|
||||||
|
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
|
#include "mediapipe/gpu/gl_base.h"
|
||||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||||
#include "mediapipe/gpu/gl_quad_renderer.h"
|
#include "mediapipe/gpu/gl_quad_renderer.h"
|
||||||
#include "mediapipe/gpu/gl_simple_shaders.h"
|
#include "mediapipe/gpu/gl_simple_shaders.h"
|
||||||
@@ -60,42 +62,42 @@ constexpr char kVideoPrestreamTag[] = "VIDEO_PRESTREAM";
|
|||||||
|
|
||||||
int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
|
int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
|
||||||
switch (rotation) {
|
switch (rotation) {
|
||||||
case mediapipe::RotationMode_Mode_UNKNOWN:
|
case mediapipe::RotationMode::UNKNOWN:
|
||||||
case mediapipe::RotationMode_Mode_ROTATION_0:
|
case mediapipe::RotationMode::ROTATION_0:
|
||||||
return 0;
|
return 0;
|
||||||
case mediapipe::RotationMode_Mode_ROTATION_90:
|
case mediapipe::RotationMode::ROTATION_90:
|
||||||
return 90;
|
return 90;
|
||||||
case mediapipe::RotationMode_Mode_ROTATION_180:
|
case mediapipe::RotationMode::ROTATION_180:
|
||||||
return 180;
|
return 180;
|
||||||
case mediapipe::RotationMode_Mode_ROTATION_270:
|
case mediapipe::RotationMode::ROTATION_270:
|
||||||
return 270;
|
return 270;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
mediapipe::RotationMode_Mode DegreesToRotationMode(int degrees) {
|
mediapipe::RotationMode_Mode DegreesToRotationMode(int degrees) {
|
||||||
switch (degrees) {
|
switch (degrees) {
|
||||||
case 0:
|
case 0:
|
||||||
return mediapipe::RotationMode_Mode_ROTATION_0;
|
return mediapipe::RotationMode::ROTATION_0;
|
||||||
case 90:
|
case 90:
|
||||||
return mediapipe::RotationMode_Mode_ROTATION_90;
|
return mediapipe::RotationMode::ROTATION_90;
|
||||||
case 180:
|
case 180:
|
||||||
return mediapipe::RotationMode_Mode_ROTATION_180;
|
return mediapipe::RotationMode::ROTATION_180;
|
||||||
case 270:
|
case 270:
|
||||||
return mediapipe::RotationMode_Mode_ROTATION_270;
|
return mediapipe::RotationMode::ROTATION_270;
|
||||||
default:
|
default:
|
||||||
return mediapipe::RotationMode_Mode_UNKNOWN;
|
return mediapipe::RotationMode::UNKNOWN;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
mediapipe::ScaleMode_Mode ParseScaleMode(
|
mediapipe::ScaleMode_Mode ParseScaleMode(
|
||||||
mediapipe::ScaleMode_Mode scale_mode,
|
mediapipe::ScaleMode_Mode scale_mode,
|
||||||
mediapipe::ScaleMode_Mode default_mode) {
|
mediapipe::ScaleMode_Mode default_mode) {
|
||||||
switch (scale_mode) {
|
switch (scale_mode) {
|
||||||
case mediapipe::ScaleMode_Mode_DEFAULT:
|
case mediapipe::ScaleMode::DEFAULT:
|
||||||
return default_mode;
|
return default_mode;
|
||||||
case mediapipe::ScaleMode_Mode_STRETCH:
|
case mediapipe::ScaleMode::STRETCH:
|
||||||
return scale_mode;
|
return scale_mode;
|
||||||
case mediapipe::ScaleMode_Mode_FIT:
|
case mediapipe::ScaleMode::FIT:
|
||||||
return scale_mode;
|
return scale_mode;
|
||||||
case mediapipe::ScaleMode_Mode_FILL_AND_CROP:
|
case mediapipe::ScaleMode::FILL_AND_CROP:
|
||||||
return scale_mode;
|
return scale_mode;
|
||||||
default:
|
default:
|
||||||
return default_mode;
|
return default_mode;
|
||||||
@@ -208,6 +210,8 @@ class ImageTransformationCalculator : public CalculatorBase {
|
|||||||
|
|
||||||
bool use_gpu_ = false;
|
bool use_gpu_ = false;
|
||||||
cv::Scalar padding_color_;
|
cv::Scalar padding_color_;
|
||||||
|
ImageTransformationCalculatorOptions::InterpolationMode interpolation_mode_;
|
||||||
|
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
GlCalculatorHelper gpu_helper_;
|
GlCalculatorHelper gpu_helper_;
|
||||||
std::unique_ptr<QuadRenderer> rgb_renderer_;
|
std::unique_ptr<QuadRenderer> rgb_renderer_;
|
||||||
@@ -343,6 +347,11 @@ absl::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
|
|||||||
options_.padding_color().green(),
|
options_.padding_color().green(),
|
||||||
options_.padding_color().blue());
|
options_.padding_color().blue());
|
||||||
|
|
||||||
|
interpolation_mode_ = options_.interpolation_mode();
|
||||||
|
if (options_.interpolation_mode() ==
|
||||||
|
ImageTransformationCalculatorOptions::DEFAULT) {
|
||||||
|
interpolation_mode_ = ImageTransformationCalculatorOptions::LINEAR;
|
||||||
|
}
|
||||||
if (use_gpu_) {
|
if (use_gpu_) {
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
// Let the helper access the GL context information.
|
// Let the helper access the GL context information.
|
||||||
@@ -457,26 +466,48 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
|
|||||||
ComputeOutputDimensions(input_width, input_height, &output_width,
|
ComputeOutputDimensions(input_width, input_height, &output_width,
|
||||||
&output_height);
|
&output_height);
|
||||||
|
|
||||||
|
int opencv_interpolation_mode = cv::INTER_LINEAR;
|
||||||
if (output_width_ > 0 && output_height_ > 0) {
|
if (output_width_ > 0 && output_height_ > 0) {
|
||||||
cv::Mat scaled_mat;
|
cv::Mat scaled_mat;
|
||||||
if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
|
if (scale_mode_ == mediapipe::ScaleMode::STRETCH) {
|
||||||
int scale_flag =
|
if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) {
|
||||||
input_mat.cols > output_width_ && input_mat.rows > output_height_
|
// Use INTER_AREA for downscaling if interpolation mode is set to
|
||||||
? cv::INTER_AREA
|
// LINEAR.
|
||||||
: cv::INTER_LINEAR;
|
if (input_mat.cols > output_width_ && input_mat.rows > output_height_) {
|
||||||
|
opencv_interpolation_mode = cv::INTER_AREA;
|
||||||
|
|
||||||
|
} else {
|
||||||
|
opencv_interpolation_mode = cv::INTER_LINEAR;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
opencv_interpolation_mode = cv::INTER_NEAREST;
|
||||||
|
}
|
||||||
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
|
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
|
||||||
0, 0, scale_flag);
|
0, 0, opencv_interpolation_mode);
|
||||||
} else {
|
} else {
|
||||||
const float scale =
|
const float scale =
|
||||||
std::min(static_cast<float>(output_width_) / input_width,
|
std::min(static_cast<float>(output_width_) / input_width,
|
||||||
static_cast<float>(output_height_) / input_height);
|
static_cast<float>(output_height_) / input_height);
|
||||||
const int target_width = std::round(input_width * scale);
|
const int target_width = std::round(input_width * scale);
|
||||||
const int target_height = std::round(input_height * scale);
|
const int target_height = std::round(input_height * scale);
|
||||||
int scale_flag = scale < 1.0f ? cv::INTER_AREA : cv::INTER_LINEAR;
|
|
||||||
if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
|
if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) {
|
||||||
|
// Use INTER_AREA for downscaling if interpolation mode is set to
|
||||||
|
// LINEAR.
|
||||||
|
if (scale < 1.0f) {
|
||||||
|
opencv_interpolation_mode = cv::INTER_AREA;
|
||||||
|
} else {
|
||||||
|
opencv_interpolation_mode = cv::INTER_LINEAR;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
opencv_interpolation_mode = cv::INTER_NEAREST;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
|
||||||
cv::Mat intermediate_mat;
|
cv::Mat intermediate_mat;
|
||||||
cv::resize(input_mat, intermediate_mat,
|
cv::resize(input_mat, intermediate_mat,
|
||||||
cv::Size(target_width, target_height), 0, 0, scale_flag);
|
cv::Size(target_width, target_height), 0, 0,
|
||||||
|
opencv_interpolation_mode);
|
||||||
const int top = (output_height_ - target_height) / 2;
|
const int top = (output_height_ - target_height) / 2;
|
||||||
const int bottom = output_height_ - target_height - top;
|
const int bottom = output_height_ - target_height - top;
|
||||||
const int left = (output_width_ - target_width) / 2;
|
const int left = (output_width_ - target_width) / 2;
|
||||||
@@ -488,7 +519,7 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
|
|||||||
padding_color_);
|
padding_color_);
|
||||||
} else {
|
} else {
|
||||||
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
|
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
|
||||||
0, 0, scale_flag);
|
0, 0, opencv_interpolation_mode);
|
||||||
output_width = target_width;
|
output_width = target_width;
|
||||||
output_height = target_height;
|
output_height = target_height;
|
||||||
}
|
}
|
||||||
@@ -514,17 +545,17 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
|
|||||||
cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size);
|
cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size);
|
||||||
} else {
|
} else {
|
||||||
switch (rotation_) {
|
switch (rotation_) {
|
||||||
case mediapipe::RotationMode_Mode_UNKNOWN:
|
case mediapipe::RotationMode::UNKNOWN:
|
||||||
case mediapipe::RotationMode_Mode_ROTATION_0:
|
case mediapipe::RotationMode::ROTATION_0:
|
||||||
rotated_mat = input_mat;
|
rotated_mat = input_mat;
|
||||||
break;
|
break;
|
||||||
case mediapipe::RotationMode_Mode_ROTATION_90:
|
case mediapipe::RotationMode::ROTATION_90:
|
||||||
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE);
|
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE);
|
||||||
break;
|
break;
|
||||||
case mediapipe::RotationMode_Mode_ROTATION_180:
|
case mediapipe::RotationMode::ROTATION_180:
|
||||||
cv::rotate(input_mat, rotated_mat, cv::ROTATE_180);
|
cv::rotate(input_mat, rotated_mat, cv::ROTATE_180);
|
||||||
break;
|
break;
|
||||||
case mediapipe::RotationMode_Mode_ROTATION_270:
|
case mediapipe::RotationMode::ROTATION_270:
|
||||||
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE);
|
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -561,7 +592,7 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
|
|||||||
ComputeOutputDimensions(input_width, input_height, &output_width,
|
ComputeOutputDimensions(input_width, input_height, &output_width,
|
||||||
&output_height);
|
&output_height);
|
||||||
|
|
||||||
if (scale_mode_ == mediapipe::ScaleMode_Mode_FILL_AND_CROP) {
|
if (scale_mode_ == mediapipe::ScaleMode::FILL_AND_CROP) {
|
||||||
const float scale =
|
const float scale =
|
||||||
std::min(static_cast<float>(output_width_) / input_width,
|
std::min(static_cast<float>(output_width_) / input_width,
|
||||||
static_cast<float>(output_height_) / input_height);
|
static_cast<float>(output_height_) / input_height);
|
||||||
@@ -628,6 +659,12 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
|
|||||||
glActiveTexture(GL_TEXTURE1);
|
glActiveTexture(GL_TEXTURE1);
|
||||||
glBindTexture(src1.target(), src1.name());
|
glBindTexture(src1.target(), src1.name());
|
||||||
|
|
||||||
|
if (interpolation_mode_ == ImageTransformationCalculatorOptions::NEAREST) {
|
||||||
|
// TODO: revert texture params.
|
||||||
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
|
||||||
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
|
||||||
|
}
|
||||||
|
|
||||||
MP_RETURN_IF_ERROR(renderer->GlRender(
|
MP_RETURN_IF_ERROR(renderer->GlRender(
|
||||||
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
|
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
|
||||||
rotation, flip_horizontally_, flip_vertically_,
|
rotation, flip_horizontally_, flip_vertically_,
|
||||||
@@ -652,8 +689,8 @@ void ImageTransformationCalculator::ComputeOutputDimensions(
|
|||||||
if (output_width_ > 0 && output_height_ > 0) {
|
if (output_width_ > 0 && output_height_ > 0) {
|
||||||
*output_width = output_width_;
|
*output_width = output_width_;
|
||||||
*output_height = output_height_;
|
*output_height = output_height_;
|
||||||
} else if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
|
} else if (rotation_ == mediapipe::RotationMode::ROTATION_90 ||
|
||||||
rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
|
rotation_ == mediapipe::RotationMode::ROTATION_270) {
|
||||||
*output_width = input_height;
|
*output_width = input_height;
|
||||||
*output_height = input_width;
|
*output_height = input_width;
|
||||||
} else {
|
} else {
|
||||||
@@ -666,9 +703,9 @@ void ImageTransformationCalculator::ComputeOutputLetterboxPadding(
|
|||||||
int input_width, int input_height, int output_width, int output_height,
|
int input_width, int input_height, int output_width, int output_height,
|
||||||
std::array<float, 4>* padding) {
|
std::array<float, 4>* padding) {
|
||||||
padding->fill(0.f);
|
padding->fill(0.f);
|
||||||
if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
|
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
|
||||||
if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
|
if (rotation_ == mediapipe::RotationMode::ROTATION_90 ||
|
||||||
rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
|
rotation_ == mediapipe::RotationMode::ROTATION_270) {
|
||||||
std::swap(input_width, input_height);
|
std::swap(input_width, input_height);
|
||||||
}
|
}
|
||||||
const float input_aspect_ratio =
|
const float input_aspect_ratio =
|
||||||
|
|||||||
@@ -54,4 +54,15 @@ message ImageTransformationCalculatorOptions {
|
|||||||
// The color for the padding. This option is only used when the scale mode is
|
// The color for the padding. This option is only used when the scale mode is
|
||||||
// FIT. Default is black. This is for CPU only.
|
// FIT. Default is black. This is for CPU only.
|
||||||
optional Color padding_color = 8;
|
optional Color padding_color = 8;
|
||||||
|
|
||||||
|
// Interpolation method to use. Note that on CPU when LINEAR is specified,
|
||||||
|
// INTER_LINEAR is used for upscaling and INTER_AREA is used for downscaling.
|
||||||
|
enum InterpolationMode {
|
||||||
|
DEFAULT = 0;
|
||||||
|
LINEAR = 1;
|
||||||
|
NEAREST = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mode DEFAULT will use LINEAR interpolation.
|
||||||
|
optional InterpolationMode interpolation_mode = 9;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,315 @@
|
|||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "absl/container/flat_hash_set.h"
|
||||||
|
#include "absl/flags/flag.h"
|
||||||
|
#include "absl/strings/substitute.h"
|
||||||
|
#include "mediapipe/framework/calculator.pb.h"
|
||||||
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/calculator_runner.h"
|
||||||
|
#include "mediapipe/framework/deps/file_path.h"
|
||||||
|
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
|
#include "mediapipe/framework/port/gtest.h"
|
||||||
|
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
|
||||||
|
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||||
|
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||||
|
#include "testing/base/public/gmock.h"
|
||||||
|
#include "testing/base/public/googletest.h"
|
||||||
|
#include "third_party/OpenCV/core.hpp" // IWYU pragma: keep
|
||||||
|
#include "third_party/OpenCV/core/mat.hpp"
|
||||||
|
|
||||||
|
namespace mediapipe {
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
absl::flat_hash_set<int> computeUniqueValues(const cv::Mat& mat) {
|
||||||
|
// Compute the unique values in cv::Mat
|
||||||
|
absl::flat_hash_set<int> unique_values;
|
||||||
|
for (int i = 0; i < mat.rows; i++) {
|
||||||
|
for (int j = 0; j < mat.cols; j++) {
|
||||||
|
unique_values.insert(mat.at<unsigned char>(i, j));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return unique_values;
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ImageTransformationCalculatorTest, NearestNeighborResizing) {
|
||||||
|
cv::Mat input_mat;
|
||||||
|
cv::cvtColor(cv::imread(file::JoinPath("./",
|
||||||
|
"/mediapipe/calculators/"
|
||||||
|
"image/testdata/binary_mask.png")),
|
||||||
|
input_mat, cv::COLOR_BGR2GRAY);
|
||||||
|
Packet input_image_packet = MakePacket<ImageFrame>(
|
||||||
|
ImageFormat::GRAY8, input_mat.size().width, input_mat.size().height);
|
||||||
|
input_mat.copyTo(formats::MatView(&(input_image_packet.Get<ImageFrame>())));
|
||||||
|
|
||||||
|
std::vector<std::pair<int, int>> output_dims{
|
||||||
|
{256, 333}, {512, 512}, {1024, 1024}};
|
||||||
|
|
||||||
|
for (auto& output_dim : output_dims) {
|
||||||
|
Packet input_output_dim_packet =
|
||||||
|
MakePacket<std::pair<int, int>>(output_dim);
|
||||||
|
std::vector<std::string> scale_modes{"FIT", "STRETCH"};
|
||||||
|
for (const auto& scale_mode : scale_modes) {
|
||||||
|
CalculatorGraphConfig::Node node_config =
|
||||||
|
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
|
||||||
|
absl::Substitute(R"(
|
||||||
|
calculator: "ImageTransformationCalculator"
|
||||||
|
input_stream: "IMAGE:input_image"
|
||||||
|
input_stream: "OUTPUT_DIMENSIONS:image_size"
|
||||||
|
output_stream: "IMAGE:output_image"
|
||||||
|
options: {
|
||||||
|
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
||||||
|
scale_mode: $0
|
||||||
|
interpolation_mode: NEAREST
|
||||||
|
}
|
||||||
|
})",
|
||||||
|
scale_mode));
|
||||||
|
|
||||||
|
CalculatorRunner runner(node_config);
|
||||||
|
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
|
||||||
|
input_image_packet.At(Timestamp(0)));
|
||||||
|
runner.MutableInputs()
|
||||||
|
->Tag("OUTPUT_DIMENSIONS")
|
||||||
|
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
|
||||||
|
|
||||||
|
MP_ASSERT_OK(runner.Run());
|
||||||
|
const auto& outputs = runner.Outputs();
|
||||||
|
ASSERT_EQ(outputs.NumEntries(), 1);
|
||||||
|
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
|
||||||
|
ASSERT_EQ(packets.size(), 1);
|
||||||
|
const auto& result = packets[0].Get<ImageFrame>();
|
||||||
|
ASSERT_EQ(output_dim.first, result.Width());
|
||||||
|
ASSERT_EQ(output_dim.second, result.Height());
|
||||||
|
|
||||||
|
auto unique_input_values = computeUniqueValues(input_mat);
|
||||||
|
auto unique_output_values =
|
||||||
|
computeUniqueValues(formats::MatView(&result));
|
||||||
|
EXPECT_THAT(unique_input_values,
|
||||||
|
::testing::ContainerEq(unique_output_values));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ImageTransformationCalculatorTest,
|
||||||
|
NearestNeighborResizingWorksForFloatInput) {
|
||||||
|
cv::Mat input_mat;
|
||||||
|
cv::cvtColor(cv::imread(file::JoinPath("./",
|
||||||
|
"/mediapipe/calculators/"
|
||||||
|
"image/testdata/binary_mask.png")),
|
||||||
|
input_mat, cv::COLOR_BGR2GRAY);
|
||||||
|
Packet input_image_packet = MakePacket<ImageFrame>(
|
||||||
|
ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
|
||||||
|
cv::Mat packet_mat_view =
|
||||||
|
formats::MatView(&(input_image_packet.Get<ImageFrame>()));
|
||||||
|
input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
|
||||||
|
|
||||||
|
std::vector<std::pair<int, int>> output_dims{
|
||||||
|
{256, 333}, {512, 512}, {1024, 1024}};
|
||||||
|
|
||||||
|
for (auto& output_dim : output_dims) {
|
||||||
|
Packet input_output_dim_packet =
|
||||||
|
MakePacket<std::pair<int, int>>(output_dim);
|
||||||
|
std::vector<std::string> scale_modes{"FIT", "STRETCH"};
|
||||||
|
for (const auto& scale_mode : scale_modes) {
|
||||||
|
CalculatorGraphConfig::Node node_config =
|
||||||
|
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
|
||||||
|
absl::Substitute(R"(
|
||||||
|
calculator: "ImageTransformationCalculator"
|
||||||
|
input_stream: "IMAGE:input_image"
|
||||||
|
input_stream: "OUTPUT_DIMENSIONS:image_size"
|
||||||
|
output_stream: "IMAGE:output_image"
|
||||||
|
options: {
|
||||||
|
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
||||||
|
scale_mode: $0
|
||||||
|
interpolation_mode: NEAREST
|
||||||
|
}
|
||||||
|
})",
|
||||||
|
scale_mode));
|
||||||
|
|
||||||
|
CalculatorRunner runner(node_config);
|
||||||
|
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
|
||||||
|
input_image_packet.At(Timestamp(0)));
|
||||||
|
runner.MutableInputs()
|
||||||
|
->Tag("OUTPUT_DIMENSIONS")
|
||||||
|
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
|
||||||
|
|
||||||
|
MP_ASSERT_OK(runner.Run());
|
||||||
|
const auto& outputs = runner.Outputs();
|
||||||
|
ASSERT_EQ(outputs.NumEntries(), 1);
|
||||||
|
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
|
||||||
|
ASSERT_EQ(packets.size(), 1);
|
||||||
|
const auto& result = packets[0].Get<ImageFrame>();
|
||||||
|
ASSERT_EQ(output_dim.first, result.Width());
|
||||||
|
ASSERT_EQ(output_dim.second, result.Height());
|
||||||
|
|
||||||
|
auto unique_input_values = computeUniqueValues(packet_mat_view);
|
||||||
|
auto unique_output_values =
|
||||||
|
computeUniqueValues(formats::MatView(&result));
|
||||||
|
EXPECT_THAT(unique_input_values,
|
||||||
|
::testing::ContainerEq(unique_output_values));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ImageTransformationCalculatorTest, NearestNeighborResizingGpu) {
|
||||||
|
cv::Mat input_mat;
|
||||||
|
cv::cvtColor(cv::imread(file::JoinPath("./",
|
||||||
|
"/mediapipe/calculators/"
|
||||||
|
"image/testdata/binary_mask.png")),
|
||||||
|
input_mat, cv::COLOR_BGR2RGBA);
|
||||||
|
|
||||||
|
std::vector<std::pair<int, int>> output_dims{
|
||||||
|
{256, 333}, {512, 512}, {1024, 1024}};
|
||||||
|
|
||||||
|
for (auto& output_dim : output_dims) {
|
||||||
|
std::vector<std::string> scale_modes{"FIT"}; //, "STRETCH"};
|
||||||
|
for (const auto& scale_mode : scale_modes) {
|
||||||
|
CalculatorGraphConfig graph_config =
|
||||||
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
|
absl::Substitute(R"(
|
||||||
|
input_stream: "input_image"
|
||||||
|
input_stream: "image_size"
|
||||||
|
output_stream: "output_image"
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "ImageFrameToGpuBufferCalculator"
|
||||||
|
input_stream: "input_image"
|
||||||
|
output_stream: "input_image_gpu"
|
||||||
|
}
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "ImageTransformationCalculator"
|
||||||
|
input_stream: "IMAGE_GPU:input_image_gpu"
|
||||||
|
input_stream: "OUTPUT_DIMENSIONS:image_size"
|
||||||
|
output_stream: "IMAGE_GPU:output_image_gpu"
|
||||||
|
options: {
|
||||||
|
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
||||||
|
scale_mode: $0
|
||||||
|
interpolation_mode: NEAREST
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
node {
|
||||||
|
calculator: "GpuBufferToImageFrameCalculator"
|
||||||
|
input_stream: "output_image_gpu"
|
||||||
|
output_stream: "output_image"
|
||||||
|
})",
|
||||||
|
scale_mode));
|
||||||
|
ImageFrame input_image(ImageFormat::SRGBA, input_mat.size().width,
|
||||||
|
input_mat.size().height);
|
||||||
|
input_mat.copyTo(formats::MatView(&input_image));
|
||||||
|
|
||||||
|
std::vector<Packet> output_image_packets;
|
||||||
|
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
|
||||||
|
|
||||||
|
CalculatorGraph graph(graph_config);
|
||||||
|
MP_ASSERT_OK(graph.StartRun({}));
|
||||||
|
|
||||||
|
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||||
|
"input_image",
|
||||||
|
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
|
||||||
|
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||||
|
"image_size",
|
||||||
|
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
|
||||||
|
|
||||||
|
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||||
|
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
|
||||||
|
|
||||||
|
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
|
||||||
|
ASSERT_EQ(output_dim.first, output_image.Width());
|
||||||
|
ASSERT_EQ(output_dim.second, output_image.Height());
|
||||||
|
|
||||||
|
auto unique_input_values = computeUniqueValues(input_mat);
|
||||||
|
auto unique_output_values =
|
||||||
|
computeUniqueValues(formats::MatView(&output_image));
|
||||||
|
EXPECT_THAT(unique_input_values,
|
||||||
|
::testing::ContainerEq(unique_output_values));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(ImageTransformationCalculatorTest,
|
||||||
|
NearestNeighborResizingWorksForFloatTexture) {
|
||||||
|
cv::Mat input_mat;
|
||||||
|
cv::cvtColor(cv::imread(file::JoinPath("./",
|
||||||
|
"/mediapipe/calculators/"
|
||||||
|
"image/testdata/binary_mask.png")),
|
||||||
|
input_mat, cv::COLOR_BGR2GRAY);
|
||||||
|
Packet input_image_packet = MakePacket<ImageFrame>(
|
||||||
|
ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
|
||||||
|
cv::Mat packet_mat_view =
|
||||||
|
formats::MatView(&(input_image_packet.Get<ImageFrame>()));
|
||||||
|
input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
|
||||||
|
|
||||||
|
std::vector<std::pair<int, int>> output_dims{
|
||||||
|
{256, 333}, {512, 512}, {1024, 1024}};
|
||||||
|
|
||||||
|
for (auto& output_dim : output_dims) {
|
||||||
|
std::vector<std::string> scale_modes{"FIT"}; //, "STRETCH"};
|
||||||
|
for (const auto& scale_mode : scale_modes) {
|
||||||
|
CalculatorGraphConfig graph_config =
|
||||||
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
|
absl::Substitute(R"(
|
||||||
|
input_stream: "input_image"
|
||||||
|
input_stream: "image_size"
|
||||||
|
output_stream: "output_image"
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "ImageFrameToGpuBufferCalculator"
|
||||||
|
input_stream: "input_image"
|
||||||
|
output_stream: "input_image_gpu"
|
||||||
|
}
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "ImageTransformationCalculator"
|
||||||
|
input_stream: "IMAGE_GPU:input_image_gpu"
|
||||||
|
input_stream: "OUTPUT_DIMENSIONS:image_size"
|
||||||
|
output_stream: "IMAGE_GPU:output_image_gpu"
|
||||||
|
options: {
|
||||||
|
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
||||||
|
scale_mode: $0
|
||||||
|
interpolation_mode: NEAREST
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
node {
|
||||||
|
calculator: "GpuBufferToImageFrameCalculator"
|
||||||
|
input_stream: "output_image_gpu"
|
||||||
|
output_stream: "output_image"
|
||||||
|
})",
|
||||||
|
scale_mode));
|
||||||
|
|
||||||
|
std::vector<Packet> output_image_packets;
|
||||||
|
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
|
||||||
|
|
||||||
|
CalculatorGraph graph(graph_config);
|
||||||
|
MP_ASSERT_OK(graph.StartRun({}));
|
||||||
|
|
||||||
|
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||||
|
"input_image", input_image_packet.At(Timestamp(0))));
|
||||||
|
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||||
|
"image_size",
|
||||||
|
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
|
||||||
|
|
||||||
|
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||||
|
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
|
||||||
|
|
||||||
|
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
|
||||||
|
ASSERT_EQ(output_dim.first, output_image.Width());
|
||||||
|
ASSERT_EQ(output_dim.second, output_image.Height());
|
||||||
|
|
||||||
|
auto unique_input_values = computeUniqueValues(packet_mat_view);
|
||||||
|
auto unique_output_values =
|
||||||
|
computeUniqueValues(formats::MatView(&output_image));
|
||||||
|
EXPECT_THAT(unique_input_values,
|
||||||
|
::testing::ContainerEq(unique_output_values));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
} // namespace mediapipe
|
||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h"
|
#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
@@ -61,7 +62,7 @@ absl::Status OpenCvImageEncoderCalculator::Open(CalculatorContext* cc) {
|
|||||||
|
|
||||||
absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) {
|
absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) {
|
||||||
const ImageFrame& image_frame = cc->Inputs().Index(0).Get<ImageFrame>();
|
const ImageFrame& image_frame = cc->Inputs().Index(0).Get<ImageFrame>();
|
||||||
CHECK_EQ(1, image_frame.ByteDepth());
|
ABSL_CHECK_EQ(1, image_frame.ByteDepth());
|
||||||
|
|
||||||
std::unique_ptr<OpenCvImageEncoderCalculatorResults> encoded_result =
|
std::unique_ptr<OpenCvImageEncoderCalculatorResults> encoded_result =
|
||||||
absl::make_unique<OpenCvImageEncoderCalculatorResults>();
|
absl::make_unique<OpenCvImageEncoderCalculatorResults>();
|
||||||
|
|||||||
@@ -18,6 +18,8 @@
|
|||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "absl/strings/str_cat.h"
|
#include "absl/strings/str_cat.h"
|
||||||
#include "absl/strings/substitute.h"
|
#include "absl/strings/substitute.h"
|
||||||
#include "libyuv/scale.h"
|
#include "libyuv/scale.h"
|
||||||
@@ -75,16 +77,16 @@ absl::Status FindInterpolationAlgorithm(
|
|||||||
|
|
||||||
void CropImageFrame(const ImageFrame& original, int col_start, int row_start,
|
void CropImageFrame(const ImageFrame& original, int col_start, int row_start,
|
||||||
int crop_width, int crop_height, ImageFrame* cropped) {
|
int crop_width, int crop_height, ImageFrame* cropped) {
|
||||||
const uint8* src = original.PixelData();
|
const uint8_t* src = original.PixelData();
|
||||||
uint8* dst = cropped->MutablePixelData();
|
uint8_t* dst = cropped->MutablePixelData();
|
||||||
|
|
||||||
int des_y = 0;
|
int des_y = 0;
|
||||||
for (int y = row_start; y < row_start + crop_height; ++y) {
|
for (int y = row_start; y < row_start + crop_height; ++y) {
|
||||||
const uint8* src_line = src + y * original.WidthStep();
|
const uint8_t* src_line = src + y * original.WidthStep();
|
||||||
const uint8* src_pixel = src_line + col_start *
|
const uint8_t* src_pixel = src_line + col_start *
|
||||||
original.NumberOfChannels() *
|
original.NumberOfChannels() *
|
||||||
original.ByteDepth();
|
original.ByteDepth();
|
||||||
uint8* dst_line = dst + des_y * cropped->WidthStep();
|
uint8_t* dst_line = dst + des_y * cropped->WidthStep();
|
||||||
std::memcpy(
|
std::memcpy(
|
||||||
dst_line, src_pixel,
|
dst_line, src_pixel,
|
||||||
crop_width * cropped->NumberOfChannels() * cropped->ByteDepth());
|
crop_width * cropped->NumberOfChannels() * cropped->ByteDepth());
|
||||||
@@ -293,7 +295,7 @@ absl::Status ScaleImageCalculator::InitializeFrameInfo(CalculatorContext* cc) {
|
|||||||
header->width = output_width_;
|
header->width = output_width_;
|
||||||
header->height = output_height_;
|
header->height = output_height_;
|
||||||
header->format = output_format_;
|
header->format = output_format_;
|
||||||
LOG(INFO) << "OUTPUTTING HEADER on stream";
|
ABSL_LOG(INFO) << "OUTPUTTING HEADER on stream";
|
||||||
cc->Outputs()
|
cc->Outputs()
|
||||||
.Tag("VIDEO_HEADER")
|
.Tag("VIDEO_HEADER")
|
||||||
.Add(header.release(), Timestamp::PreStream());
|
.Add(header.release(), Timestamp::PreStream());
|
||||||
@@ -393,10 +395,11 @@ absl::Status ScaleImageCalculator::Open(CalculatorContext* cc) {
|
|||||||
.SetHeader(Adopt(output_header.release()));
|
.SetHeader(Adopt(output_header.release()));
|
||||||
has_header_ = true;
|
has_header_ = true;
|
||||||
} else {
|
} else {
|
||||||
LOG(WARNING) << "Stream had a VideoHeader which didn't have sufficient "
|
ABSL_LOG(WARNING)
|
||||||
"information. "
|
<< "Stream had a VideoHeader which didn't have sufficient "
|
||||||
"Dropping VideoHeader and trying to deduce needed "
|
"information. "
|
||||||
"information.";
|
"Dropping VideoHeader and trying to deduce needed "
|
||||||
|
"information.";
|
||||||
input_width_ = 0;
|
input_width_ = 0;
|
||||||
input_height_ = 0;
|
input_height_ = 0;
|
||||||
if (!options_.has_input_format()) {
|
if (!options_.has_input_format()) {
|
||||||
@@ -507,7 +510,7 @@ absl::Status ScaleImageCalculator::ValidateImageFrame(
|
|||||||
|
|
||||||
absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc,
|
absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc,
|
||||||
const YUVImage& yuv_image) {
|
const YUVImage& yuv_image) {
|
||||||
CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
|
ABSL_CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
|
||||||
if (!has_header_) {
|
if (!has_header_) {
|
||||||
if (input_width_ != yuv_image.width() ||
|
if (input_width_ != yuv_image.width() ||
|
||||||
input_height_ != yuv_image.height()) {
|
input_height_ != yuv_image.height()) {
|
||||||
@@ -591,9 +594,9 @@ absl::Status ScaleImageCalculator::Process(CalculatorContext* cc) {
|
|||||||
const int y_size = output_width_ * output_height_;
|
const int y_size = output_width_ * output_height_;
|
||||||
const int uv_size = output_width_ * output_height_ / 4;
|
const int uv_size = output_width_ * output_height_ / 4;
|
||||||
std::unique_ptr<uint8_t[]> yuv_data(new uint8_t[y_size + uv_size * 2]);
|
std::unique_ptr<uint8_t[]> yuv_data(new uint8_t[y_size + uv_size * 2]);
|
||||||
uint8* y = yuv_data.get();
|
uint8_t* y = yuv_data.get();
|
||||||
uint8* u = y + y_size;
|
uint8_t* u = y + y_size;
|
||||||
uint8* v = u + uv_size;
|
uint8_t* v = u + uv_size;
|
||||||
RET_CHECK_EQ(0, I420Scale(yuv_image->data(0), yuv_image->stride(0),
|
RET_CHECK_EQ(0, I420Scale(yuv_image->data(0), yuv_image->stride(0),
|
||||||
yuv_image->data(1), yuv_image->stride(1),
|
yuv_image->data(1), yuv_image->stride(1),
|
||||||
yuv_image->data(2), yuv_image->stride(2),
|
yuv_image->data(2), yuv_image->stride(2),
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include "absl/log/absl_check.h"
|
||||||
#include "absl/strings/str_split.h"
|
#include "absl/strings/str_split.h"
|
||||||
#include "mediapipe/framework/port/logging.h"
|
#include "mediapipe/framework/port/logging.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
@@ -40,10 +41,10 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
|
|||||||
const std::string& max_aspect_ratio, //
|
const std::string& max_aspect_ratio, //
|
||||||
int* crop_width, int* crop_height, //
|
int* crop_width, int* crop_height, //
|
||||||
int* col_start, int* row_start) {
|
int* col_start, int* row_start) {
|
||||||
CHECK(crop_width);
|
ABSL_CHECK(crop_width);
|
||||||
CHECK(crop_height);
|
ABSL_CHECK(crop_height);
|
||||||
CHECK(col_start);
|
ABSL_CHECK(col_start);
|
||||||
CHECK(row_start);
|
ABSL_CHECK(row_start);
|
||||||
|
|
||||||
double min_aspect_ratio_q = 0.0;
|
double min_aspect_ratio_q = 0.0;
|
||||||
double max_aspect_ratio_q = 0.0;
|
double max_aspect_ratio_q = 0.0;
|
||||||
@@ -83,8 +84,8 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
CHECK_LE(*crop_width, input_width);
|
ABSL_CHECK_LE(*crop_width, input_width);
|
||||||
CHECK_LE(*crop_height, input_height);
|
ABSL_CHECK_LE(*crop_height, input_height);
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -96,8 +97,8 @@ absl::Status FindOutputDimensions(int input_width, //
|
|||||||
bool preserve_aspect_ratio, //
|
bool preserve_aspect_ratio, //
|
||||||
int scale_to_multiple_of, //
|
int scale_to_multiple_of, //
|
||||||
int* output_width, int* output_height) {
|
int* output_width, int* output_height) {
|
||||||
CHECK(output_width);
|
ABSL_CHECK(output_width);
|
||||||
CHECK(output_height);
|
ABSL_CHECK(output_height);
|
||||||
|
|
||||||
if (target_max_area > 0 && input_width * input_height > target_max_area) {
|
if (target_max_area > 0 && input_width * input_height > target_max_area) {
|
||||||
preserve_aspect_ratio = true;
|
preserve_aspect_ratio = true;
|
||||||
|
|||||||
@@ -15,13 +15,13 @@
|
|||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
|
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_options.pb.h"
|
#include "mediapipe/framework/calculator_options.pb.h"
|
||||||
#include "mediapipe/framework/formats/image.h"
|
#include "mediapipe/framework/formats/image.h"
|
||||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||||
#include "mediapipe/framework/formats/image_frame.h"
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
#include "mediapipe/framework/port/logging.h"
|
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
#include "mediapipe/framework/port/vector.h"
|
#include "mediapipe/framework/port/vector.h"
|
||||||
|
|
||||||
@@ -110,7 +110,7 @@ REGISTER_CALCULATOR(SegmentationSmoothingCalculator);
|
|||||||
|
|
||||||
absl::Status SegmentationSmoothingCalculator::GetContract(
|
absl::Status SegmentationSmoothingCalculator::GetContract(
|
||||||
CalculatorContract* cc) {
|
CalculatorContract* cc) {
|
||||||
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||||
|
|
||||||
cc->Inputs().Tag(kCurrentMaskTag).Set<Image>();
|
cc->Inputs().Tag(kCurrentMaskTag).Set<Image>();
|
||||||
cc->Inputs().Tag(kPreviousMaskTag).Set<Image>();
|
cc->Inputs().Tag(kPreviousMaskTag).Set<Image>();
|
||||||
@@ -273,7 +273,7 @@ absl::Status SegmentationSmoothingCalculator::RenderGpu(CalculatorContext* cc) {
|
|||||||
|
|
||||||
const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get<Image>();
|
const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get<Image>();
|
||||||
if (previous_frame.format() != current_frame.format()) {
|
if (previous_frame.format() != current_frame.format()) {
|
||||||
LOG(ERROR) << "Warning: mixing input format types. ";
|
ABSL_LOG(ERROR) << "Warning: mixing input format types. ";
|
||||||
}
|
}
|
||||||
auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame);
|
auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame);
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@
|
|||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
|
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_runner.h"
|
#include "mediapipe/framework/calculator_runner.h"
|
||||||
@@ -169,7 +170,7 @@ void RunTest(bool use_gpu, float mix_ratio, cv::Mat& test_result) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
LOG(ERROR) << "invalid ratio";
|
ABSL_LOG(ERROR) << "invalid ratio";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,13 +14,13 @@
|
|||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
|
#include "absl/log/absl_log.h"
|
||||||
#include "mediapipe/calculators/image/set_alpha_calculator.pb.h"
|
#include "mediapipe/calculators/image/set_alpha_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_options.pb.h"
|
#include "mediapipe/framework/calculator_options.pb.h"
|
||||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||||
#include "mediapipe/framework/formats/image_frame.h"
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
#include "mediapipe/framework/port/logging.h"
|
|
||||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
@@ -142,7 +142,7 @@ class SetAlphaCalculator : public CalculatorBase {
|
|||||||
REGISTER_CALCULATOR(SetAlphaCalculator);
|
REGISTER_CALCULATOR(SetAlphaCalculator);
|
||||||
|
|
||||||
absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
|
absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
|
||||||
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||||
|
|
||||||
bool use_gpu = false;
|
bool use_gpu = false;
|
||||||
|
|
||||||
@@ -268,7 +268,7 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
|
|||||||
const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
|
const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
|
||||||
const cv::Mat input_mat = formats::MatView(&input_frame);
|
const cv::Mat input_mat = formats::MatView(&input_frame);
|
||||||
if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) {
|
if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) {
|
||||||
LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
|
ABSL_LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
|
||||||
}
|
}
|
||||||
|
|
||||||
// Setup destination image
|
// Setup destination image
|
||||||
@@ -328,7 +328,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
|
|||||||
cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
|
cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
|
||||||
if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 ||
|
if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 ||
|
||||||
input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) {
|
input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) {
|
||||||
LOG(ERROR) << "Only RGB or RGBA input image supported";
|
ABSL_LOG(ERROR) << "Only RGB or RGBA input image supported";
|
||||||
}
|
}
|
||||||
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
|
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
|
||||||
|
|
||||||
|
|||||||
+1
@@ -18,6 +18,7 @@ licenses(["notice"])
|
|||||||
filegroup(
|
filegroup(
|
||||||
name = "test_images",
|
name = "test_images",
|
||||||
srcs = [
|
srcs = [
|
||||||
|
"binary_mask.png",
|
||||||
"dino.jpg",
|
"dino.jpg",
|
||||||
"dino_quality_50.jpg",
|
"dino_quality_50.jpg",
|
||||||
"dino_quality_80.jpg",
|
"dino_quality_80.jpg",
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 771 B |
@@ -79,8 +79,8 @@ class WarpAffineRunnerHolder<ImageFrame> {
|
|||||||
}
|
}
|
||||||
absl::StatusOr<RunnerType*> GetRunner() {
|
absl::StatusOr<RunnerType*> GetRunner() {
|
||||||
if (!runner_) {
|
if (!runner_) {
|
||||||
ASSIGN_OR_RETURN(runner_,
|
MP_ASSIGN_OR_RETURN(
|
||||||
CreateAffineTransformationOpenCvRunner(interpolation_));
|
runner_, CreateAffineTransformationOpenCvRunner(interpolation_));
|
||||||
}
|
}
|
||||||
return runner_.get();
|
return runner_.get();
|
||||||
}
|
}
|
||||||
@@ -108,8 +108,9 @@ class WarpAffineRunnerHolder<mediapipe::GpuBuffer> {
|
|||||||
}
|
}
|
||||||
absl::StatusOr<RunnerType*> GetRunner() {
|
absl::StatusOr<RunnerType*> GetRunner() {
|
||||||
if (!runner_) {
|
if (!runner_) {
|
||||||
ASSIGN_OR_RETURN(runner_, CreateAffineTransformationGlRunner(
|
MP_ASSIGN_OR_RETURN(
|
||||||
gl_helper_, gpu_origin_, interpolation_));
|
runner_, CreateAffineTransformationGlRunner(gl_helper_, gpu_origin_,
|
||||||
|
interpolation_));
|
||||||
}
|
}
|
||||||
return runner_.get();
|
return runner_.get();
|
||||||
}
|
}
|
||||||
@@ -151,24 +152,25 @@ class WarpAffineRunnerHolder<mediapipe::Image> {
|
|||||||
AffineTransformation::BorderMode border_mode) override {
|
AffineTransformation::BorderMode border_mode) override {
|
||||||
if (input.UsesGpu()) {
|
if (input.UsesGpu()) {
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
ASSIGN_OR_RETURN(auto* runner, gpu_holder_.GetRunner());
|
MP_ASSIGN_OR_RETURN(auto* runner, gpu_holder_.GetRunner());
|
||||||
ASSIGN_OR_RETURN(auto result, runner->Run(input.GetGpuBuffer(), matrix,
|
MP_ASSIGN_OR_RETURN(
|
||||||
size, border_mode));
|
auto result,
|
||||||
|
runner->Run(input.GetGpuBuffer(), matrix, size, border_mode));
|
||||||
return mediapipe::Image(*result);
|
return mediapipe::Image(*result);
|
||||||
#else
|
#else
|
||||||
return absl::UnavailableError("GPU support is disabled");
|
return absl::UnavailableError("GPU support is disabled");
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
}
|
}
|
||||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||||
ASSIGN_OR_RETURN(auto* runner, cpu_holder_.GetRunner());
|
MP_ASSIGN_OR_RETURN(auto* runner, cpu_holder_.GetRunner());
|
||||||
const auto& frame_ptr = input.GetImageFrameSharedPtr();
|
const auto& frame_ptr = input.GetImageFrameSharedPtr();
|
||||||
// Wrap image into image frame.
|
// Wrap image into image frame.
|
||||||
const ImageFrame image_frame(frame_ptr->Format(), frame_ptr->Width(),
|
const ImageFrame image_frame(frame_ptr->Format(), frame_ptr->Width(),
|
||||||
frame_ptr->Height(), frame_ptr->WidthStep(),
|
frame_ptr->Height(), frame_ptr->WidthStep(),
|
||||||
const_cast<uint8_t*>(frame_ptr->PixelData()),
|
const_cast<uint8_t*>(frame_ptr->PixelData()),
|
||||||
[](uint8* data){});
|
[](uint8_t* data){});
|
||||||
ASSIGN_OR_RETURN(auto result,
|
MP_ASSIGN_OR_RETURN(auto result,
|
||||||
runner->Run(image_frame, matrix, size, border_mode));
|
runner->Run(image_frame, matrix, size, border_mode));
|
||||||
return mediapipe::Image(std::make_shared<ImageFrame>(std::move(result)));
|
return mediapipe::Image(std::make_shared<ImageFrame>(std::move(result)));
|
||||||
#else
|
#else
|
||||||
return absl::UnavailableError("OpenCV support is disabled");
|
return absl::UnavailableError("OpenCV support is disabled");
|
||||||
@@ -213,8 +215,8 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
|
|||||||
AffineTransformation::Size output_size;
|
AffineTransformation::Size output_size;
|
||||||
output_size.width = out_width;
|
output_size.width = out_width;
|
||||||
output_size.height = out_height;
|
output_size.height = out_height;
|
||||||
ASSIGN_OR_RETURN(auto* runner, holder_.GetRunner());
|
MP_ASSIGN_OR_RETURN(auto* runner, holder_.GetRunner());
|
||||||
ASSIGN_OR_RETURN(
|
MP_ASSIGN_OR_RETURN(
|
||||||
auto result,
|
auto result,
|
||||||
runner->Run(
|
runner->Run(
|
||||||
*InterfaceT::kInImage(cc), transform, output_size,
|
*InterfaceT::kInImage(cc), transform, output_size,
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ void RunTest(const std::string& graph_text, const std::string& tag,
|
|||||||
|
|
||||||
ImageFrame input_image(
|
ImageFrame input_image(
|
||||||
input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB,
|
input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB,
|
||||||
input.cols, input.rows, input.step, input.data, [](uint8*) {});
|
input.cols, input.rows, input.step, input.data, [](uint8_t*) {});
|
||||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||||
"input_image",
|
"input_image",
|
||||||
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
|
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
|
|||||||
buf[0] = (fourcc >> 24) & 0xff;
|
buf[0] = (fourcc >> 24) & 0xff;
|
||||||
buf[1] = (fourcc >> 16) & 0xff;
|
buf[1] = (fourcc >> 16) & 0xff;
|
||||||
buf[2] = (fourcc >> 8) & 0xff;
|
buf[2] = (fourcc >> 8) & 0xff;
|
||||||
buf[3] = (fourcc)&0xff;
|
buf[3] = (fourcc) & 0xff;
|
||||||
buf[4] = 0;
|
buf[4] = 0;
|
||||||
return std::string(buf);
|
return std::string(buf);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,12 +31,14 @@ mediapipe_proto_library(
|
|||||||
cc_library(
|
cc_library(
|
||||||
name = "callback_packet_calculator",
|
name = "callback_packet_calculator",
|
||||||
srcs = ["callback_packet_calculator.cc"],
|
srcs = ["callback_packet_calculator.cc"],
|
||||||
|
hdrs = ["callback_packet_calculator.h"],
|
||||||
visibility = ["//mediapipe/framework:__subpackages__"],
|
visibility = ["//mediapipe/framework:__subpackages__"],
|
||||||
deps = [
|
deps = [
|
||||||
":callback_packet_calculator_cc_proto",
|
":callback_packet_calculator_cc_proto",
|
||||||
"//mediapipe/framework:calculator_base",
|
"//mediapipe/framework:calculator_base",
|
||||||
"//mediapipe/framework:calculator_registry",
|
"//mediapipe/framework:calculator_registry",
|
||||||
"//mediapipe/framework:output_side_packet",
|
"//mediapipe/framework:output_side_packet",
|
||||||
|
"@com_google_absl//absl/status",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -11,10 +11,12 @@
|
|||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
#include "mediapipe/calculators/internal/callback_packet_calculator.h"
|
||||||
|
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include "absl/status/status.h"
|
||||||
#include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT
|
#include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT
|
||||||
#include "mediapipe/framework/calculator_base.h"
|
#include "mediapipe/framework/calculator_base.h"
|
||||||
#include "mediapipe/framework/calculator_registry.h"
|
#include "mediapipe/framework/calculator_registry.h"
|
||||||
@@ -39,64 +41,55 @@ void DumpPostStreamPacket(Packet* post_stream_packet, const Packet& packet) {
|
|||||||
*post_stream_packet = packet;
|
*post_stream_packet = packet;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
// Creates a callback which takes a packet and stores it either in a
|
absl::Status CallbackPacketCalculator::GetContract(CalculatorContract* cc) {
|
||||||
// vector of packets or stores only the packet at PostStream timestamp.
|
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
|
||||||
// The kind of callback is controlled by an option. The callback is
|
switch (options.type()) {
|
||||||
// a std::function and is directly usable by CallbackCalculator.
|
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
|
||||||
// Since the options for the packet generator include a serialized pointer
|
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
|
||||||
// value, the resulting callback is only valid on the original machine
|
cc->OutputSidePackets()
|
||||||
// while that pointer is still alive.
|
.Index(0)
|
||||||
class CallbackPacketCalculator : public CalculatorBase {
|
.Set<std::function<void(const Packet&)>>();
|
||||||
public:
|
break;
|
||||||
static absl::Status GetContract(CalculatorContract* cc) {
|
default:
|
||||||
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
|
|
||||||
switch (options.type()) {
|
|
||||||
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
|
|
||||||
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
|
|
||||||
cc->OutputSidePackets()
|
|
||||||
.Index(0)
|
|
||||||
.Set<std::function<void(const Packet&)>>();
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
|
||||||
<< "Invalid type of callback to produce.";
|
|
||||||
}
|
|
||||||
return absl::OkStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
absl::Status Open(CalculatorContext* cc) override {
|
|
||||||
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
|
|
||||||
void* ptr;
|
|
||||||
if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
|
|
||||||
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||||
<< "Stored pointer value in options is invalid.";
|
<< "Invalid type of callback to produce.";
|
||||||
}
|
|
||||||
switch (options.type()) {
|
|
||||||
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
|
|
||||||
cc->OutputSidePackets().Index(0).Set(
|
|
||||||
MakePacket<std::function<void(const Packet&)>>(std::bind(
|
|
||||||
&DumpToVector, reinterpret_cast<std::vector<Packet>*>(ptr),
|
|
||||||
std::placeholders::_1)));
|
|
||||||
break;
|
|
||||||
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
|
|
||||||
cc->OutputSidePackets().Index(0).Set(
|
|
||||||
MakePacket<std::function<void(const Packet&)>>(
|
|
||||||
std::bind(&DumpPostStreamPacket, reinterpret_cast<Packet*>(ptr),
|
|
||||||
std::placeholders::_1)));
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
|
||||||
<< "Invalid type to dump into.";
|
|
||||||
}
|
|
||||||
return absl::OkStatus();
|
|
||||||
}
|
}
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
absl::Status Process(CalculatorContext* cc) override {
|
absl::Status CallbackPacketCalculator::Open(CalculatorContext* cc) {
|
||||||
return absl::OkStatus();
|
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
|
||||||
|
void* ptr;
|
||||||
|
if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
|
||||||
|
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||||
|
<< "Stored pointer value in options is invalid.";
|
||||||
}
|
}
|
||||||
};
|
switch (options.type()) {
|
||||||
|
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
|
||||||
|
cc->OutputSidePackets().Index(0).Set(
|
||||||
|
MakePacket<std::function<void(const Packet&)>>(std::bind(
|
||||||
|
&DumpToVector, reinterpret_cast<std::vector<Packet>*>(ptr),
|
||||||
|
std::placeholders::_1)));
|
||||||
|
break;
|
||||||
|
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
|
||||||
|
cc->OutputSidePackets().Index(0).Set(
|
||||||
|
MakePacket<std::function<void(const Packet&)>>(
|
||||||
|
std::bind(&DumpPostStreamPacket, reinterpret_cast<Packet*>(ptr),
|
||||||
|
std::placeholders::_1)));
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||||
|
<< "Invalid type to dump into.";
|
||||||
|
}
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status CallbackPacketCalculator::Process(CalculatorContext* cc) {
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
REGISTER_CALCULATOR(CallbackPacketCalculator);
|
REGISTER_CALCULATOR(CallbackPacketCalculator);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
// Copyright 2023 The MediaPipe Authors.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#ifndef MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
|
||||||
|
#define MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
|
||||||
|
|
||||||
|
#include "absl/status/status.h"
|
||||||
|
#include "mediapipe/framework/calculator_base.h"
|
||||||
|
|
||||||
|
namespace mediapipe {
|
||||||
|
|
||||||
|
// Creates a callback which takes a packet and stores it either in a
|
||||||
|
// vector of packets or stores only the packet at PostStream timestamp.
|
||||||
|
// The kind of callback is controlled by an option. The callback is
|
||||||
|
// a std::function and is directly usable by CallbackCalculator.
|
||||||
|
// Since the options for the packet generator include a serialized pointer
|
||||||
|
// value, the resulting callback is only valid on the original machine
|
||||||
|
// while that pointer is still alive.
|
||||||
|
class CallbackPacketCalculator : public CalculatorBase {
|
||||||
|
public:
|
||||||
|
static absl::Status GetContract(CalculatorContract* cc);
|
||||||
|
absl::Status Open(CalculatorContext* cc) override;
|
||||||
|
absl::Status Process(CalculatorContext* cc) override;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace mediapipe
|
||||||
|
|
||||||
|
#endif // MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user