Compare commits
@@ -87,6 +87,9 @@ build:ios_fat --config=ios
|
||||
build:ios_fat --ios_multi_cpus=armv7,arm64
|
||||
build:ios_fat --watchos_cpus=armv7k
|
||||
|
||||
build:ios_sim_fat --config=ios
|
||||
build:ios_sim_fat --ios_multi_cpus=x86_64,sim_arm64
|
||||
|
||||
build:darwin_x86_64 --apple_platform_type=macos
|
||||
build:darwin_x86_64 --macos_minimum_os=10.12
|
||||
build:darwin_x86_64 --cpu=darwin_x86_64
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
# Copyright 2021 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
#
|
||||
# This file was assembled from multiple pieces, whose use is documented
|
||||
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||
# for more information.
|
||||
|
||||
# Number of days of inactivity before an Issue or Pull Request becomes stale
|
||||
daysUntilStale: 7
|
||||
# Number of days of inactivity before a stale Issue or Pull Request is closed
|
||||
daysUntilClose: 7
|
||||
# Only issues or pull requests with all of these labels are checked if stale. Defaults to `[]` (disabled)
|
||||
onlyLabels:
|
||||
- stat:awaiting response
|
||||
# Comment to post when marking as stale. Set to `false` to disable
|
||||
markComment: >
|
||||
This issue has been automatically marked as stale because it has not had
|
||||
recent activity. It will be closed if no further activity occurs. Thank you.
|
||||
# Comment to post when removing the stale label. Set to `false` to disable
|
||||
unmarkComment: false
|
||||
closeComment: >
|
||||
Closing as stale. Please reopen if you'd like to work on this further.
|
||||
@@ -0,0 +1,66 @@
|
||||
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
# This workflow alerts and then closes the stale issues/PRs after specific time
|
||||
# You can adjust the behavior by modifying this file.
|
||||
# For more information, see:
|
||||
# https://github.com/actions/stale
|
||||
|
||||
name: 'Close stale issues and PRs'
|
||||
"on":
|
||||
schedule:
|
||||
- cron: "30 1 * * *"
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: 'actions/stale@v7'
|
||||
with:
|
||||
# Comma separated list of labels that can be assigned to issues to exclude them from being marked as stale.
|
||||
exempt-issue-labels: 'override-stale'
|
||||
# Comma separated list of labels that can be assigned to PRs to exclude them from being marked as stale.
|
||||
exempt-pr-labels: "override-stale"
|
||||
# Limit the No. of API calls in one run default value is 30.
|
||||
operations-per-run: 500
|
||||
# Prevent to remove stale label when PRs or issues are updated.
|
||||
remove-stale-when-updated: false
|
||||
# comment on issue if not active for more then 7 days.
|
||||
stale-issue-message: 'This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.'
|
||||
# comment on PR if not active for more then 14 days.
|
||||
stale-pr-message: 'This PR has been marked stale because it has no recent activity since 14 days. It will be closed if no further activity occurs. Thank you.'
|
||||
# comment on issue if stale for more then 7 days.
|
||||
close-issue-message: This issue was closed due to lack of activity after being marked stale for past 7 days.
|
||||
# comment on PR if stale for more then 14 days.
|
||||
close-pr-message: This PR was closed due to lack of activity after being marked stale for past 14 days.
|
||||
# Number of days of inactivity before an Issue Request becomes stale
|
||||
days-before-issue-stale: 7
|
||||
# Number of days of inactivity before a stale Issue is closed
|
||||
days-before-issue-close: 7
|
||||
# reason for closed the issue default value is not_planned
|
||||
close-issue-reason: completed
|
||||
# Number of days of inactivity before a stale PR is closed
|
||||
days-before-pr-close: 14
|
||||
# Number of days of inactivity before an PR Request becomes stale
|
||||
days-before-pr-stale: 14
|
||||
# Check for label to stale or close the issue/PR
|
||||
any-of-labels: 'stat:awaiting response'
|
||||
# override stale to stalled for PR
|
||||
stale-pr-label: 'stale'
|
||||
# override stale to stalled for Issue
|
||||
stale-issue-label: "stale"
|
||||
@@ -1,99 +1,121 @@
|
||||
---
|
||||
layout: default
|
||||
layout: forward
|
||||
target: https://developers.google.com/mediapipe
|
||||
title: Home
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
|
||||
|
||||
----
|
||||
|
||||
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
|
||||
**Attention:** *We have moved to
|
||||
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
|
||||
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
||||
|
||||
*This notice and web page will be removed on June 1, 2023.*
|
||||
|
||||
|
||||
----
|
||||
**Attention**: MediaPipe Solutions Preview is an early release. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/about#notice).
|
||||
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
**On-device machine learning for everyone**
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
Delight your customers with innovative machine learning features. MediaPipe
|
||||
contains everything that you need to customize and deploy to mobile (Android,
|
||||
iOS), web, desktop, edge devices, and IoT, effortlessly.
|
||||
|
||||
## Live ML anywhere
|
||||
* [See demos](https://goo.gle/mediapipe-studio)
|
||||
* [Learn more](https://developers.google.com/mediapipe/solutions)
|
||||
|
||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
||||
ML solutions for live and streaming media.
|
||||
## Get started
|
||||
|
||||
 | 
|
||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
 | 
|
||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
||||
You can get started with MediaPipe Solutions by by checking out any of the
|
||||
developer guides for
|
||||
[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
|
||||
[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
|
||||
and
|
||||
[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
|
||||
tasks. If you need help setting up a development environment for use with
|
||||
MediaPipe Tasks, check out the setup guides for
|
||||
[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
|
||||
apps](https://developers.google.com/mediapipe/solutions/setup_web), and
|
||||
[Python](https://developers.google.com/mediapipe/solutions/setup_python).
|
||||
|
||||
----
|
||||
## Solutions
|
||||
|
||||
## ML solutions in MediaPipe
|
||||
MediaPipe Solutions provides a suite of libraries and tools for you to quickly
|
||||
apply artificial intelligence (AI) and machine learning (ML) techniques in your
|
||||
applications. You can plug these solutions into your applications immediately,
|
||||
customize them to your needs, and use them across multiple development
|
||||
platforms. MediaPipe Solutions is part of the MediaPipe [open source
|
||||
project](https://github.com/google/mediapipe), so you can further customize the
|
||||
solutions code to meet your application needs.
|
||||
|
||||
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
|
||||
These libraries and resources provide the core functionality for each MediaPipe
|
||||
Solution:
|
||||
|
||||
Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
||||
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
|
||||
solutions. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/tasks).
|
||||
* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
|
||||
solution.
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
||||
These tools let you customize and evaluate solutions:
|
||||
|
||||
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
||||
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
* **MediaPipe Model Maker**: Customize models for solutions with your data.
|
||||
[Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
|
||||
* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
|
||||
browser. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/studio).
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
### Legacy solutions
|
||||
|
||||
## Getting started
|
||||
We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
|
||||
a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
|
||||
and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
|
||||
provided on an as-is basis.
|
||||
|
||||
To start using MediaPipe
|
||||
[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
|
||||
lines code, see example code and demos in
|
||||
[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
|
||||
[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
|
||||
For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
|
||||
|
||||
To use MediaPipe in C++, Android and iOS, which allow further customization of
|
||||
the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
|
||||
building your own, learn how to
|
||||
[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
|
||||
start building example applications in
|
||||
[C++](https://google.github.io/mediapipe/getting_started/cpp),
|
||||
[Android](https://google.github.io/mediapipe/getting_started/android) and
|
||||
[iOS](https://google.github.io/mediapipe/getting_started/ios).
|
||||
## Framework
|
||||
|
||||
The source code is hosted in the
|
||||
[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
|
||||
run code search using
|
||||
[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
|
||||
To start using MediaPipe Framework, [install MediaPipe
|
||||
Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
|
||||
and start building example applications in C++, Android, and iOS.
|
||||
|
||||
## Publications
|
||||
[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
|
||||
low-level component used to build efficient on-device machine learning
|
||||
pipelines, similar to the premade MediaPipe Solutions.
|
||||
|
||||
Before using MediaPipe Framework, familiarize yourself with the following key
|
||||
[Framework
|
||||
concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
|
||||
|
||||
* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
|
||||
* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
|
||||
* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
|
||||
|
||||
## Community
|
||||
|
||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
|
||||
users.
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||
community discussion around MediaPipe.
|
||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||
curated list of awesome MediaPipe related frameworks, libraries and
|
||||
software.
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions. Please follow these
|
||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||
|
||||
## Resources
|
||||
|
||||
### Publications
|
||||
|
||||
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
||||
in Google Developers Blog
|
||||
@@ -102,7 +124,8 @@ run code search using
|
||||
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
||||
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
||||
in Google Developers Blog
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
|
||||
Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
in Google AI Blog
|
||||
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
||||
in Google AI Blog
|
||||
@@ -130,43 +153,6 @@ run code search using
|
||||
in Google AI Blog
|
||||
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
||||
|
||||
## Videos
|
||||
### Videos
|
||||
|
||||
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
|
||||
|
||||
## Events
|
||||
|
||||
* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
|
||||
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
|
||||
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
|
||||
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
|
||||
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
|
||||
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
|
||||
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
|
||||
Seoul, Korea ICCV
|
||||
2019](https://research.google.com/youtube8m/workshop2019/index.html)
|
||||
* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
|
||||
* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
|
||||
([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
|
||||
* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
|
||||
|
||||
## Community
|
||||
|
||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||
curated list of awesome MediaPipe related frameworks, libraries and software
|
||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||
community discussion around MediaPipe
|
||||
|
||||
## Alpha disclaimer
|
||||
|
||||
MediaPipe is currently in alpha at v0.7. We may be still making breaking API
|
||||
changes and expect to get to stable APIs by v1.0.
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions. Please follow these
|
||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||
|
||||
@@ -45,12 +45,13 @@ http_archive(
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "rules_foreign_cc",
|
||||
strip_prefix = "rules_foreign_cc-0.1.0",
|
||||
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
|
||||
name = "rules_foreign_cc",
|
||||
sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
|
||||
strip_prefix = "rules_foreign_cc-0.9.0",
|
||||
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
|
||||
)
|
||||
|
||||
load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
|
||||
load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
|
||||
|
||||
rules_foreign_cc_dependencies()
|
||||
|
||||
@@ -156,22 +157,22 @@ http_archive(
|
||||
# 2020-08-21
|
||||
http_archive(
|
||||
name = "com_github_glog_glog",
|
||||
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
|
||||
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
|
||||
strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
|
||||
sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
|
||||
urls = [
|
||||
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
|
||||
"https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
|
||||
],
|
||||
)
|
||||
http_archive(
|
||||
name = "com_github_glog_glog_no_gflags",
|
||||
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
|
||||
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
|
||||
strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
|
||||
sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
|
||||
build_file = "@//third_party:glog_no_gflags.BUILD",
|
||||
urls = [
|
||||
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
|
||||
"https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
|
||||
],
|
||||
patches = [
|
||||
"@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff",
|
||||
"@//third_party:com_github_glog_glog.diff",
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
@@ -266,10 +267,10 @@ http_archive(
|
||||
|
||||
http_archive(
|
||||
name = "com_googlesource_code_re2",
|
||||
sha256 = "e06b718c129f4019d6e7aa8b7631bee38d3d450dd980246bfaf493eb7db67868",
|
||||
strip_prefix = "re2-fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8",
|
||||
sha256 = "ef516fb84824a597c4d5d0d6d330daedb18363b5a99eda87d027e6bdd9cba299",
|
||||
strip_prefix = "re2-03da4fc0857c285e3a26782f6bc8931c4c950df4",
|
||||
urls = [
|
||||
"https://github.com/google/re2/archive/fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8.tar.gz",
|
||||
"https://github.com/google/re2/archive/03da4fc0857c285e3a26782f6bc8931c4c950df4.tar.gz",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -375,6 +376,22 @@ http_archive(
|
||||
url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
|
||||
)
|
||||
|
||||
# Building an opencv.xcframework from the OpenCV 4.5.3 sources is necessary for
|
||||
# MediaPipe iOS Task Libraries to be supported on arm64(M1) Macs. An
|
||||
# `opencv.xcframework` archive has not been released and it is recommended to
|
||||
# build the same from source using a script provided in OpenCV 4.5.0 upwards.
|
||||
# OpenCV is fixed to version to 4.5.3 since swift support can only be disabled
|
||||
# from 4.5.3 upwards. This is needed to avoid errors when the library is linked
|
||||
# in Xcode. Swift support will be added in when the final binary MediaPipe iOS
|
||||
# Task libraries are built.
|
||||
http_archive(
|
||||
name = "ios_opencv_source",
|
||||
sha256 = "a61e7a4618d353140c857f25843f39b2abe5f451b018aab1604ef0bc34cd23d5",
|
||||
build_file = "@//third_party:opencv_ios_source.BUILD",
|
||||
type = "zip",
|
||||
url = "https://github.com/opencv/opencv/archive/refs/tags/4.5.3.zip",
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "stblib",
|
||||
strip_prefix = "stb-b42009b3b9d4ca35bc703f5310eedc74f584be58",
|
||||
@@ -468,9 +485,10 @@ http_archive(
|
||||
)
|
||||
|
||||
# TensorFlow repo should always go after the other external dependencies.
|
||||
# TF on 2023-04-12.
|
||||
_TENSORFLOW_GIT_COMMIT = "d712c0c9e24519cc8cd3720279666720d1000eee"
|
||||
_TENSORFLOW_SHA256 = "ba98de6ea5f720071246691a1536ecd5e1b1763033e8c82a1e721a06d3dfd4c1"
|
||||
# TF on 2023-06-13.
|
||||
_TENSORFLOW_GIT_COMMIT = "491681a5620e41bf079a582ac39c585cc86878b9"
|
||||
# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
|
||||
_TENSORFLOW_SHA256 = "9f76389af7a2835e68413322c1eaabfadc912f02a76d71dc16be507f9ca3d3ac"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
urls = [
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
@@ -14,6 +14,7 @@
|
||||
# ==============================================================================
|
||||
"""Generate Java reference docs for MediaPipe."""
|
||||
import pathlib
|
||||
import shutil
|
||||
|
||||
from absl import app
|
||||
from absl import flags
|
||||
@@ -41,7 +42,9 @@ def main(_) -> None:
|
||||
mp_root = pathlib.Path(__file__)
|
||||
while (mp_root := mp_root.parent).name != 'mediapipe':
|
||||
# Find the nearest `mediapipe` dir.
|
||||
pass
|
||||
if not mp_root.name:
|
||||
# We've hit the filesystem root - abort.
|
||||
raise FileNotFoundError('"mediapipe" root not found')
|
||||
|
||||
# Find the root from which all packages are relative.
|
||||
root = mp_root.parent
|
||||
@@ -51,6 +54,14 @@ def main(_) -> None:
|
||||
if (mp_root / 'mediapipe').exists():
|
||||
mp_root = mp_root / 'mediapipe'
|
||||
|
||||
# We need to copy this into the tasks dir to ensure we don't leave broken
|
||||
# links in the generated docs.
|
||||
old_api_dir = 'java/com/google/mediapipe/framework/image'
|
||||
shutil.copytree(
|
||||
mp_root / old_api_dir,
|
||||
mp_root / 'tasks' / old_api_dir,
|
||||
dirs_exist_ok=True)
|
||||
|
||||
gen_java.gen_java_docs(
|
||||
package='com.google.mediapipe',
|
||||
source_path=mp_root / 'tasks/java',
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
+93
-107
@@ -1,99 +1,121 @@
|
||||
---
|
||||
layout: default
|
||||
layout: forward
|
||||
target: https://developers.google.com/mediapipe
|
||||
title: Home
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
|
||||
|
||||
----
|
||||
|
||||
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
|
||||
**Attention:** *We have moved to
|
||||
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
|
||||
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
||||
|
||||
*This notice and web page will be removed on June 1, 2023.*
|
||||
|
||||
|
||||
----
|
||||
**Attention**: MediaPipe Solutions Preview is an early release. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/about#notice).
|
||||
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
**On-device machine learning for everyone**
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
Delight your customers with innovative machine learning features. MediaPipe
|
||||
contains everything that you need to customize and deploy to mobile (Android,
|
||||
iOS), web, desktop, edge devices, and IoT, effortlessly.
|
||||
|
||||
## Live ML anywhere
|
||||
* [See demos](https://goo.gle/mediapipe-studio)
|
||||
* [Learn more](https://developers.google.com/mediapipe/solutions)
|
||||
|
||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
||||
ML solutions for live and streaming media.
|
||||
## Get started
|
||||
|
||||
 | 
|
||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
 | 
|
||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
||||
You can get started with MediaPipe Solutions by by checking out any of the
|
||||
developer guides for
|
||||
[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
|
||||
[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
|
||||
and
|
||||
[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
|
||||
tasks. If you need help setting up a development environment for use with
|
||||
MediaPipe Tasks, check out the setup guides for
|
||||
[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
|
||||
apps](https://developers.google.com/mediapipe/solutions/setup_web), and
|
||||
[Python](https://developers.google.com/mediapipe/solutions/setup_python).
|
||||
|
||||
----
|
||||
## Solutions
|
||||
|
||||
## ML solutions in MediaPipe
|
||||
MediaPipe Solutions provides a suite of libraries and tools for you to quickly
|
||||
apply artificial intelligence (AI) and machine learning (ML) techniques in your
|
||||
applications. You can plug these solutions into your applications immediately,
|
||||
customize them to your needs, and use them across multiple development
|
||||
platforms. MediaPipe Solutions is part of the MediaPipe [open source
|
||||
project](https://github.com/google/mediapipe), so you can further customize the
|
||||
solutions code to meet your application needs.
|
||||
|
||||
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
|
||||
These libraries and resources provide the core functionality for each MediaPipe
|
||||
Solution:
|
||||
|
||||
Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
||||
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
|
||||
solutions. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/tasks).
|
||||
* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
|
||||
solution.
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
||||
These tools let you customize and evaluate solutions:
|
||||
|
||||
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
||||
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
* **MediaPipe Model Maker**: Customize models for solutions with your data.
|
||||
[Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
|
||||
* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
|
||||
browser. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/studio).
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
### Legacy solutions
|
||||
|
||||
## Getting started
|
||||
We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
|
||||
a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
|
||||
and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
|
||||
provided on an as-is basis.
|
||||
|
||||
To start using MediaPipe
|
||||
[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
|
||||
lines code, see example code and demos in
|
||||
[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
|
||||
[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
|
||||
For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
|
||||
|
||||
To use MediaPipe in C++, Android and iOS, which allow further customization of
|
||||
the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
|
||||
building your own, learn how to
|
||||
[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
|
||||
start building example applications in
|
||||
[C++](https://google.github.io/mediapipe/getting_started/cpp),
|
||||
[Android](https://google.github.io/mediapipe/getting_started/android) and
|
||||
[iOS](https://google.github.io/mediapipe/getting_started/ios).
|
||||
## Framework
|
||||
|
||||
The source code is hosted in the
|
||||
[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
|
||||
run code search using
|
||||
[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
|
||||
To start using MediaPipe Framework, [install MediaPipe
|
||||
Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
|
||||
and start building example applications in C++, Android, and iOS.
|
||||
|
||||
## Publications
|
||||
[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
|
||||
low-level component used to build efficient on-device machine learning
|
||||
pipelines, similar to the premade MediaPipe Solutions.
|
||||
|
||||
Before using MediaPipe Framework, familiarize yourself with the following key
|
||||
[Framework
|
||||
concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
|
||||
|
||||
* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
|
||||
* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
|
||||
* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
|
||||
|
||||
## Community
|
||||
|
||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
|
||||
users.
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||
community discussion around MediaPipe.
|
||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||
curated list of awesome MediaPipe related frameworks, libraries and
|
||||
software.
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions. Please follow these
|
||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||
|
||||
## Resources
|
||||
|
||||
### Publications
|
||||
|
||||
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
||||
in Google Developers Blog
|
||||
@@ -102,7 +124,8 @@ run code search using
|
||||
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
||||
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
||||
in Google Developers Blog
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
|
||||
Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
in Google AI Blog
|
||||
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
||||
in Google AI Blog
|
||||
@@ -130,43 +153,6 @@ run code search using
|
||||
in Google AI Blog
|
||||
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
||||
|
||||
## Videos
|
||||
### Videos
|
||||
|
||||
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
|
||||
|
||||
## Events
|
||||
|
||||
* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
|
||||
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
|
||||
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
|
||||
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
|
||||
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
|
||||
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
|
||||
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
|
||||
Seoul, Korea ICCV
|
||||
2019](https://research.google.com/youtube8m/workshop2019/index.html)
|
||||
* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
|
||||
* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
|
||||
([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
|
||||
* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
|
||||
|
||||
## Community
|
||||
|
||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||
curated list of awesome MediaPipe related frameworks, libraries and software
|
||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||
community discussion around MediaPipe
|
||||
|
||||
## Alpha disclaimer
|
||||
|
||||
MediaPipe is currently in alpha at v0.7. We may be still making breaking API
|
||||
changes and expect to get to stable APIs by v1.0.
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions. Please follow these
|
||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||
|
||||
@@ -20,9 +20,9 @@ nav_order: 1
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_detector)
|
||||
site.*
|
||||
|
||||
----
|
||||
|
||||
@@ -20,9 +20,9 @@ nav_order: 2
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
|
||||
site.*
|
||||
|
||||
----
|
||||
|
||||
@@ -20,9 +20,9 @@ nav_order: 3
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
|
||||
site.*
|
||||
|
||||
----
|
||||
|
||||
@@ -22,9 +22,9 @@ nav_order: 5
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker)
|
||||
site.*
|
||||
|
||||
----
|
||||
|
||||
@@ -21,7 +21,7 @@ nav_order: 1
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
|
||||
site.*
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
---
|
||||
layout: default
|
||||
layout: forward
|
||||
target: https://developers.google.com/mediapipe/solutions/guide#legacy
|
||||
title: MediaPipe Legacy Solutions
|
||||
nav_order: 3
|
||||
has_children: true
|
||||
@@ -13,8 +14,7 @@ has_toc: false
|
||||
{:toc}
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions. We have
|
||||
ended support for
|
||||
**Attention:** *We have ended support for
|
||||
[these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
as of March 1, 2023. All other
|
||||
[MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
@@ -25,14 +25,6 @@ be provided on an as-is basis. We encourage you to check out the new MediaPipe
|
||||
Solutions at:
|
||||
[https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
|
||||
|
||||
*This notice and web page will be removed on June 1, 2023.*
|
||||
|
||||
----
|
||||
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
----
|
||||
|
||||
MediaPipe offers open source cross-platform, customizable ML solutions for live
|
||||
|
||||
+93
-57
@@ -68,30 +68,108 @@ config_setting(
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# Note: this cannot just match "apple_platform_type": "macos" because that option
|
||||
# defaults to "macos" even when building on Linux!
|
||||
alias(
|
||||
# Generic MacOS.
|
||||
config_setting(
|
||||
name = "macos",
|
||||
actual = select({
|
||||
":macos_i386": ":macos_i386",
|
||||
":macos_x86_64": ":macos_x86_64",
|
||||
":macos_arm64": ":macos_arm64",
|
||||
"//conditions:default": ":macos_i386", # Arbitrarily chosen from above.
|
||||
}),
|
||||
constraint_values = [
|
||||
"@platforms//os:macos",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# Note: this also matches on crosstool_top so that it does not produce ambiguous
|
||||
# selectors when used together with "android".
|
||||
# MacOS x86 64-bit.
|
||||
config_setting(
|
||||
name = "macos_x86_64",
|
||||
constraint_values = [
|
||||
"@platforms//os:macos",
|
||||
"@platforms//cpu:x86_64",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# MacOS ARM64.
|
||||
config_setting(
|
||||
name = "macos_arm64",
|
||||
constraint_values = [
|
||||
"@platforms//os:macos",
|
||||
"@platforms//cpu:arm64",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# Generic iOS.
|
||||
config_setting(
|
||||
name = "ios",
|
||||
values = {
|
||||
"crosstool_top": "@bazel_tools//tools/cpp:toolchain",
|
||||
"apple_platform_type": "ios",
|
||||
},
|
||||
constraint_values = [
|
||||
"@platforms//os:ios",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# iOS device ARM32.
|
||||
config_setting(
|
||||
name = "ios_armv7",
|
||||
constraint_values = [
|
||||
"@platforms//os:ios",
|
||||
"@platforms//cpu:arm",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# iOS device ARM64.
|
||||
config_setting(
|
||||
name = "ios_arm64",
|
||||
constraint_values = [
|
||||
"@platforms//os:ios",
|
||||
"@platforms//cpu:arm64",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# iOS device ARM64E.
|
||||
config_setting(
|
||||
name = "ios_arm64e",
|
||||
constraint_values = [
|
||||
"@platforms//os:ios",
|
||||
"@platforms//cpu:arm64e",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# iOS simulator x86 32-bit.
|
||||
config_setting(
|
||||
name = "ios_i386",
|
||||
constraint_values = [
|
||||
"@platforms//os:ios",
|
||||
"@platforms//cpu:x86_32",
|
||||
"@build_bazel_apple_support//constraints:simulator",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# iOS simulator x86 64-bit.
|
||||
config_setting(
|
||||
name = "ios_x86_64",
|
||||
constraint_values = [
|
||||
"@platforms//os:ios",
|
||||
"@platforms//cpu:x86_64",
|
||||
"@build_bazel_apple_support//constraints:simulator",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# iOS simulator ARM64.
|
||||
config_setting(
|
||||
name = "ios_sim_arm64",
|
||||
constraint_values = [
|
||||
"@platforms//os:ios",
|
||||
"@platforms//cpu:arm64",
|
||||
"@build_bazel_apple_support//constraints:simulator",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# Generic Apple.
|
||||
alias(
|
||||
name = "apple",
|
||||
actual = select({
|
||||
@@ -102,48 +180,6 @@ alias(
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "macos_i386",
|
||||
values = {
|
||||
"apple_platform_type": "macos",
|
||||
"cpu": "darwin",
|
||||
},
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "macos_x86_64",
|
||||
values = {
|
||||
"apple_platform_type": "macos",
|
||||
"cpu": "darwin_x86_64",
|
||||
},
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "macos_arm64",
|
||||
values = {
|
||||
"apple_platform_type": "macos",
|
||||
"cpu": "darwin_arm64",
|
||||
},
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
[
|
||||
config_setting(
|
||||
name = arch,
|
||||
values = {"cpu": arch},
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
for arch in [
|
||||
"ios_i386",
|
||||
"ios_x86_64",
|
||||
"ios_armv7",
|
||||
"ios_arm64",
|
||||
"ios_arm64e",
|
||||
]
|
||||
]
|
||||
|
||||
config_setting(
|
||||
name = "windows",
|
||||
values = {"cpu": "x64_windows"},
|
||||
|
||||
@@ -219,12 +219,10 @@ cc_library(
|
||||
deps = [
|
||||
":time_series_framer_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:time_series_util",
|
||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||
"@eigen_archive//:eigen3",
|
||||
@@ -319,6 +317,20 @@ cc_test(
|
||||
],
|
||||
)
|
||||
|
||||
cc_binary(
|
||||
name = "time_series_framer_calculator_benchmark",
|
||||
srcs = ["time_series_framer_calculator_benchmark.cc"],
|
||||
deps = [
|
||||
":time_series_framer_calculator",
|
||||
":time_series_framer_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"@com_google_benchmark//:benchmark",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "time_series_framer_calculator_test",
|
||||
srcs = ["time_series_framer_calculator_test.cc"],
|
||||
|
||||
@@ -210,6 +210,23 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
|
||||
// Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
|
||||
const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;
|
||||
|
||||
namespace {
|
||||
std::unique_ptr<audio_dsp::WindowFunction> MakeWindowFun(
|
||||
const SpectrogramCalculatorOptions::WindowType window_type) {
|
||||
switch (window_type) {
|
||||
// The cosine window and square root of Hann are equivalent.
|
||||
case SpectrogramCalculatorOptions::COSINE:
|
||||
case SpectrogramCalculatorOptions::SQRT_HANN:
|
||||
return std::make_unique<audio_dsp::CosineWindow>();
|
||||
case SpectrogramCalculatorOptions::HANN:
|
||||
return std::make_unique<audio_dsp::HannWindow>();
|
||||
case SpectrogramCalculatorOptions::HAMMING:
|
||||
return std::make_unique<audio_dsp::HammingWindow>();
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
|
||||
SpectrogramCalculatorOptions spectrogram_options =
|
||||
cc->Options<SpectrogramCalculatorOptions>();
|
||||
@@ -266,28 +283,14 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
|
||||
|
||||
output_scale_ = spectrogram_options.output_scale();
|
||||
|
||||
std::vector<double> window;
|
||||
switch (spectrogram_options.window_type()) {
|
||||
case SpectrogramCalculatorOptions::COSINE:
|
||||
audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
|
||||
&window);
|
||||
break;
|
||||
case SpectrogramCalculatorOptions::HANN:
|
||||
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
|
||||
&window);
|
||||
break;
|
||||
case SpectrogramCalculatorOptions::HAMMING:
|
||||
audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
|
||||
&window);
|
||||
break;
|
||||
case SpectrogramCalculatorOptions::SQRT_HANN: {
|
||||
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
|
||||
&window);
|
||||
absl::c_transform(window, window.begin(),
|
||||
[](double x) { return std::sqrt(x); });
|
||||
break;
|
||||
}
|
||||
auto window_fun = MakeWindowFun(spectrogram_options.window_type());
|
||||
if (window_fun == nullptr) {
|
||||
return absl::Status(absl::StatusCode::kInvalidArgument,
|
||||
absl::StrCat("Invalid window type ",
|
||||
spectrogram_options.window_type()));
|
||||
}
|
||||
std::vector<double> window;
|
||||
window_fun->GetPeriodicSamples(frame_duration_samples_, &window);
|
||||
|
||||
// Propagate settings down to the actual Spectrogram object.
|
||||
spectrogram_generators_.clear();
|
||||
@@ -433,9 +436,9 @@ absl::Status SpectrogramCalculator::ProcessVectorToOutput(
|
||||
absl::Status SpectrogramCalculator::ProcessVector(const Matrix& input_stream,
|
||||
CalculatorContext* cc) {
|
||||
switch (output_type_) {
|
||||
// These blocks deliberately ignore clang-format to preserve the
|
||||
// "silhouette" of the different cases.
|
||||
// clang-format off
|
||||
// These blocks deliberately ignore clang-format to preserve the
|
||||
// "silhouette" of the different cases.
|
||||
// clang-format off
|
||||
case SpectrogramCalculatorOptions::COMPLEX: {
|
||||
return ProcessVectorToOutput(
|
||||
input_stream,
|
||||
|
||||
@@ -68,7 +68,7 @@ message SpectrogramCalculatorOptions {
|
||||
HANN = 0;
|
||||
HAMMING = 1;
|
||||
COSINE = 2;
|
||||
SQRT_HANN = 4;
|
||||
SQRT_HANN = 4; // Alias of COSINE.
|
||||
}
|
||||
optional WindowType window_type = 6 [default = HANN];
|
||||
|
||||
|
||||
@@ -15,9 +15,7 @@
|
||||
// Defines TimeSeriesFramerCalculator.
|
||||
#include <math.h>
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Eigen/Core"
|
||||
#include "audio/dsp/window_functions.h"
|
||||
@@ -25,9 +23,8 @@
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/util/time_series_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
@@ -88,11 +85,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
|
||||
absl::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Adds input data to the internal buffer.
|
||||
void EnqueueInput(CalculatorContext* cc);
|
||||
// Constructs and emits framed output packets.
|
||||
void FrameOutput(CalculatorContext* cc);
|
||||
|
||||
Timestamp CurrentOutputTimestamp() {
|
||||
if (use_local_timestamp_) {
|
||||
return current_timestamp_;
|
||||
@@ -106,14 +98,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
|
||||
Timestamp::kTimestampUnitsPerSecond);
|
||||
}
|
||||
|
||||
// Returns the timestamp of a sample on a base, which is usually the time
|
||||
// stamp of a packet.
|
||||
Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
|
||||
int64_t number_of_samples) {
|
||||
return timestamp_base + round(number_of_samples / sample_rate_ *
|
||||
Timestamp::kTimestampUnitsPerSecond);
|
||||
}
|
||||
|
||||
// The number of input samples to advance after the current output frame is
|
||||
// emitted.
|
||||
int next_frame_step_samples() const {
|
||||
@@ -142,61 +126,174 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
|
||||
Timestamp initial_input_timestamp_;
|
||||
// The current timestamp is updated along with the incoming packets.
|
||||
Timestamp current_timestamp_;
|
||||
int num_channels_;
|
||||
|
||||
// Each entry in this deque consists of a single sample, i.e. a
|
||||
// single column vector, and its timestamp.
|
||||
std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
|
||||
// Samples are buffered in a vector of sample blocks.
|
||||
class SampleBlockBuffer {
|
||||
public:
|
||||
// Initializes the buffer.
|
||||
void Init(double sample_rate, int num_channels) {
|
||||
ts_units_per_sample_ = Timestamp::kTimestampUnitsPerSecond / sample_rate;
|
||||
num_channels_ = num_channels;
|
||||
num_samples_ = 0;
|
||||
first_block_offset_ = 0;
|
||||
}
|
||||
|
||||
// Number of channels, equal to the number of rows in each Matrix.
|
||||
int num_channels() const { return num_channels_; }
|
||||
// Total number of available samples over all blocks.
|
||||
int num_samples() const { return num_samples_; }
|
||||
|
||||
// Pushes a new block of samples on the back of the buffer with `timestamp`
|
||||
// being the input timestamp of the packet containing the Matrix.
|
||||
void Push(const Matrix& samples, Timestamp timestamp);
|
||||
// Copies `count` samples from the front of the buffer. If there are fewer
|
||||
// samples than this, the result is zero padded to have `count` samples.
|
||||
// The timestamp of the last copied sample is written to *last_timestamp.
|
||||
// This output is used below to update `current_timestamp_`, which is only
|
||||
// used when `use_local_timestamp` is true.
|
||||
Matrix CopySamples(int count, Timestamp* last_timestamp) const;
|
||||
// Drops `count` samples from the front of the buffer. If `count` exceeds
|
||||
// `num_samples()`, the buffer is emptied. Returns how many samples were
|
||||
// dropped.
|
||||
int DropSamples(int count);
|
||||
|
||||
private:
|
||||
struct Block {
|
||||
// Matrix of num_channels rows by num_samples columns, a block of possibly
|
||||
// multiple samples.
|
||||
Matrix samples;
|
||||
// Timestamp of the first sample in the Block. This comes from the input
|
||||
// packet's timestamp that contains this Matrix.
|
||||
Timestamp timestamp;
|
||||
|
||||
Block() : timestamp(Timestamp::Unstarted()) {}
|
||||
Block(const Matrix& samples, Timestamp timestamp)
|
||||
: samples(samples), timestamp(timestamp) {}
|
||||
int num_samples() const { return samples.cols(); }
|
||||
};
|
||||
std::vector<Block> blocks_;
|
||||
// Number of timestamp units per sample. Used to compute timestamps as
|
||||
// nth sample timestamp = base_timestamp + round(ts_units_per_sample_ * n).
|
||||
double ts_units_per_sample_;
|
||||
// Number of rows in each Matrix.
|
||||
int num_channels_;
|
||||
// The total number of samples over all blocks, equal to
|
||||
// (sum_i blocks_[i].num_samples()) - first_block_offset_.
|
||||
int num_samples_;
|
||||
// The number of samples in the first block that have been discarded. This
|
||||
// way we can cheaply represent "partially discarding" a block.
|
||||
int first_block_offset_;
|
||||
} sample_buffer_;
|
||||
|
||||
bool use_window_;
|
||||
Matrix window_;
|
||||
Eigen::RowVectorXf window_;
|
||||
|
||||
bool use_local_timestamp_;
|
||||
};
|
||||
REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
|
||||
|
||||
void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
|
||||
const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
|
||||
|
||||
for (int i = 0; i < input_frame.cols(); ++i) {
|
||||
sample_buffer_.emplace_back(std::make_pair(
|
||||
input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
|
||||
}
|
||||
void TimeSeriesFramerCalculator::SampleBlockBuffer::Push(const Matrix& samples,
|
||||
Timestamp timestamp) {
|
||||
num_samples_ += samples.cols();
|
||||
blocks_.emplace_back(samples, timestamp);
|
||||
}
|
||||
|
||||
void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
|
||||
while (sample_buffer_.size() >=
|
||||
Matrix TimeSeriesFramerCalculator::SampleBlockBuffer::CopySamples(
|
||||
int count, Timestamp* last_timestamp) const {
|
||||
Matrix copied(num_channels_, count);
|
||||
|
||||
if (!blocks_.empty()) {
|
||||
int num_copied = 0;
|
||||
// First block has an offset for samples that have been discarded.
|
||||
int offset = first_block_offset_;
|
||||
int n;
|
||||
Timestamp last_block_ts;
|
||||
int last_sample_index;
|
||||
|
||||
for (auto it = blocks_.begin(); it != blocks_.end() && count > 0; ++it) {
|
||||
n = std::min(it->num_samples() - offset, count);
|
||||
// Copy `n` samples from the next block.
|
||||
copied.middleCols(num_copied, n) = it->samples.middleCols(offset, n);
|
||||
count -= n;
|
||||
num_copied += n;
|
||||
last_block_ts = it->timestamp;
|
||||
last_sample_index = offset + n - 1;
|
||||
offset = 0; // No samples have been discarded in subsequent blocks.
|
||||
}
|
||||
|
||||
// Compute the timestamp of the last copied sample.
|
||||
*last_timestamp =
|
||||
last_block_ts + std::round(ts_units_per_sample_ * last_sample_index);
|
||||
}
|
||||
|
||||
if (count > 0) {
|
||||
copied.rightCols(count).setZero(); // Zero pad if needed.
|
||||
}
|
||||
|
||||
return copied;
|
||||
}
|
||||
|
||||
int TimeSeriesFramerCalculator::SampleBlockBuffer::DropSamples(int count) {
|
||||
if (blocks_.empty()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
auto block_it = blocks_.begin();
|
||||
if (first_block_offset_ + count < block_it->num_samples()) {
|
||||
// `count` is less than the remaining samples in the first block.
|
||||
first_block_offset_ += count;
|
||||
num_samples_ -= count;
|
||||
return count;
|
||||
}
|
||||
|
||||
int num_samples_dropped = block_it->num_samples() - first_block_offset_;
|
||||
count -= num_samples_dropped;
|
||||
first_block_offset_ = 0;
|
||||
|
||||
for (++block_it; block_it != blocks_.end(); ++block_it) {
|
||||
if (block_it->num_samples() > count) {
|
||||
break;
|
||||
}
|
||||
num_samples_dropped += block_it->num_samples();
|
||||
count -= block_it->num_samples();
|
||||
}
|
||||
|
||||
blocks_.erase(blocks_.begin(), block_it); // Drop whole blocks.
|
||||
if (!blocks_.empty()) {
|
||||
first_block_offset_ = count; // Drop part of the next block.
|
||||
num_samples_dropped += count;
|
||||
}
|
||||
|
||||
num_samples_ -= num_samples_dropped;
|
||||
return num_samples_dropped;
|
||||
}
|
||||
|
||||
absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
|
||||
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
|
||||
initial_input_timestamp_ = cc->InputTimestamp();
|
||||
current_timestamp_ = initial_input_timestamp_;
|
||||
}
|
||||
|
||||
// Add input data to the internal buffer.
|
||||
sample_buffer_.Push(cc->Inputs().Index(0).Get<Matrix>(),
|
||||
cc->InputTimestamp());
|
||||
|
||||
// Construct and emit framed output packets.
|
||||
while (sample_buffer_.num_samples() >=
|
||||
frame_duration_samples_ + samples_still_to_drop_) {
|
||||
while (samples_still_to_drop_ > 0) {
|
||||
sample_buffer_.pop_front();
|
||||
--samples_still_to_drop_;
|
||||
}
|
||||
sample_buffer_.DropSamples(samples_still_to_drop_);
|
||||
Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
|
||||
¤t_timestamp_);
|
||||
const int frame_step_samples = next_frame_step_samples();
|
||||
std::unique_ptr<Matrix> output_frame(
|
||||
new Matrix(num_channels_, frame_duration_samples_));
|
||||
for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
|
||||
++i) {
|
||||
output_frame->col(i) = sample_buffer_.front().first;
|
||||
current_timestamp_ = sample_buffer_.front().second;
|
||||
sample_buffer_.pop_front();
|
||||
}
|
||||
const int frame_overlap_samples =
|
||||
frame_duration_samples_ - frame_step_samples;
|
||||
if (frame_overlap_samples > 0) {
|
||||
for (int i = 0; i < frame_overlap_samples; ++i) {
|
||||
output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
|
||||
current_timestamp_ = sample_buffer_[i].second;
|
||||
}
|
||||
} else {
|
||||
samples_still_to_drop_ = -frame_overlap_samples;
|
||||
}
|
||||
samples_still_to_drop_ = frame_step_samples;
|
||||
|
||||
if (use_window_) {
|
||||
*output_frame = (output_frame->array() * window_.array()).matrix();
|
||||
// Apply the window to each row of output_frame.
|
||||
output_frame.array().rowwise() *= window_.array();
|
||||
}
|
||||
|
||||
cc->Outputs().Index(0).Add(output_frame.release(),
|
||||
CurrentOutputTimestamp());
|
||||
cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
|
||||
.At(CurrentOutputTimestamp()));
|
||||
++cumulative_output_frames_;
|
||||
cumulative_completed_samples_ += frame_step_samples;
|
||||
}
|
||||
@@ -206,35 +303,18 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
|
||||
// fact to enable packet queueing optimizations.
|
||||
cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
|
||||
}
|
||||
}
|
||||
|
||||
absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
|
||||
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
|
||||
initial_input_timestamp_ = cc->InputTimestamp();
|
||||
current_timestamp_ = initial_input_timestamp_;
|
||||
}
|
||||
|
||||
EnqueueInput(cc);
|
||||
FrameOutput(cc);
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
|
||||
while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
|
||||
sample_buffer_.pop_front();
|
||||
--samples_still_to_drop_;
|
||||
}
|
||||
if (!sample_buffer_.empty() && pad_final_packet_) {
|
||||
std::unique_ptr<Matrix> output_frame(new Matrix);
|
||||
output_frame->setZero(num_channels_, frame_duration_samples_);
|
||||
for (int i = 0; i < sample_buffer_.size(); ++i) {
|
||||
output_frame->col(i) = sample_buffer_[i].first;
|
||||
current_timestamp_ = sample_buffer_[i].second;
|
||||
}
|
||||
sample_buffer_.DropSamples(samples_still_to_drop_);
|
||||
|
||||
cc->Outputs().Index(0).Add(output_frame.release(),
|
||||
CurrentOutputTimestamp());
|
||||
if (sample_buffer_.num_samples() > 0 && pad_final_packet_) {
|
||||
Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
|
||||
¤t_timestamp_);
|
||||
cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
|
||||
.At(CurrentOutputTimestamp()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
@@ -258,7 +338,7 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
|
||||
cc->Inputs().Index(0).Header(), &input_header));
|
||||
|
||||
sample_rate_ = input_header.sample_rate();
|
||||
num_channels_ = input_header.num_channels();
|
||||
sample_buffer_.Init(sample_rate_, input_header.num_channels());
|
||||
frame_duration_samples_ = time_series_util::SecondsToSamples(
|
||||
framer_options.frame_duration_seconds(), sample_rate_);
|
||||
RET_CHECK_GT(frame_duration_samples_, 0)
|
||||
@@ -312,9 +392,8 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
|
||||
}
|
||||
|
||||
if (use_window_) {
|
||||
window_ = Matrix::Ones(num_channels_, 1) *
|
||||
Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
|
||||
frame_duration_samples_)
|
||||
window_ = Eigen::Map<Eigen::RowVectorXd>(window_vector.data(),
|
||||
frame_duration_samples_)
|
||||
.cast<float>();
|
||||
}
|
||||
use_local_timestamp_ = framer_options.use_local_timestamp();
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Benchmark for TimeSeriesFramerCalculator.
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
#include "benchmark/benchmark.h"
|
||||
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
|
||||
using ::mediapipe::Matrix;
|
||||
|
||||
void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
|
||||
constexpr float kSampleRate = 32000.0;
|
||||
constexpr int kNumChannels = 2;
|
||||
constexpr int kFrameDurationSeconds = 5.0;
|
||||
std::mt19937 rng(0 /*seed*/);
|
||||
// Input around a half second's worth of samples at a time.
|
||||
std::uniform_int_distribution<int> input_size_dist(15000, 17000);
|
||||
// Generate a pool of random blocks of samples up front.
|
||||
std::vector<Matrix> sample_pool;
|
||||
sample_pool.reserve(20);
|
||||
for (int i = 0; i < 20; ++i) {
|
||||
sample_pool.push_back(Matrix::Random(kNumChannels, input_size_dist(rng)));
|
||||
}
|
||||
std::uniform_int_distribution<int> pool_index_dist(0, sample_pool.size() - 1);
|
||||
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
config.add_input_stream("input");
|
||||
config.add_output_stream("output");
|
||||
auto* node = config.add_node();
|
||||
node->set_calculator("TimeSeriesFramerCalculator");
|
||||
node->add_input_stream("input");
|
||||
node->add_output_stream("output");
|
||||
mediapipe::TimeSeriesFramerCalculatorOptions* options =
|
||||
node->mutable_options()->MutableExtension(
|
||||
mediapipe::TimeSeriesFramerCalculatorOptions::ext);
|
||||
options->set_frame_duration_seconds(kFrameDurationSeconds);
|
||||
|
||||
for (auto _ : state) {
|
||||
state.PauseTiming(); // Pause benchmark timing.
|
||||
|
||||
// Prepare input packets of random blocks of samples.
|
||||
std::vector<mediapipe::Packet> input_packets;
|
||||
input_packets.reserve(32);
|
||||
float t = 0;
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
auto samples =
|
||||
std::make_unique<Matrix>(sample_pool[pool_index_dist(rng)]);
|
||||
const int num_samples = samples->cols();
|
||||
input_packets.push_back(mediapipe::Adopt(samples.release())
|
||||
.At(mediapipe::Timestamp::FromSeconds(t)));
|
||||
t += num_samples / kSampleRate;
|
||||
}
|
||||
// Initialize graph.
|
||||
mediapipe::CalculatorGraph graph;
|
||||
CHECK_OK(graph.Initialize(config));
|
||||
// Prepare input header.
|
||||
auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
|
||||
header->set_sample_rate(kSampleRate);
|
||||
header->set_num_channels(kNumChannels);
|
||||
|
||||
state.ResumeTiming(); // Resume benchmark timing.
|
||||
|
||||
CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
|
||||
for (auto& packet : input_packets) {
|
||||
CHECK_OK(graph.AddPacketToInputStream("input", packet));
|
||||
}
|
||||
CHECK(!graph.HasError());
|
||||
CHECK_OK(graph.CloseAllInputStreams());
|
||||
CHECK_OK(graph.WaitUntilIdle());
|
||||
}
|
||||
}
|
||||
BENCHMARK(BM_TimeSeriesFramerCalculator);
|
||||
|
||||
BENCHMARK_MAIN();
|
||||
@@ -117,6 +117,7 @@ mediapipe_proto_library(
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/framework/formats:classification_proto",
|
||||
"//mediapipe/framework/formats:landmark_proto",
|
||||
"//mediapipe/framework/formats:matrix_data_proto",
|
||||
"//mediapipe/framework/formats:time_series_header_proto",
|
||||
],
|
||||
)
|
||||
@@ -192,17 +193,19 @@ cc_library(
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -215,18 +218,18 @@ cc_library(
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/util:render_data_cc_proto",
|
||||
"@com_google_absl//absl/status",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
],
|
||||
alwayslink = 1,
|
||||
@@ -287,6 +290,7 @@ cc_library(
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/api2:port",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
@@ -295,8 +299,7 @@ cc_library(
|
||||
"//mediapipe/util:render_data_cc_proto",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//mediapipe:ios": [],
|
||||
":ios_or_disable_gpu": [],
|
||||
"//conditions:default": [
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||
],
|
||||
@@ -378,17 +381,6 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "clip_detection_vector_size_calculator",
|
||||
srcs = ["clip_detection_vector_size_calculator.cc"],
|
||||
deps = [
|
||||
":clip_vector_size_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "clip_vector_size_calculator_test",
|
||||
srcs = ["clip_vector_size_calculator_test.cc"],
|
||||
@@ -904,6 +896,7 @@ cc_library(
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
@@ -1136,6 +1129,7 @@ cc_library(
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
@@ -1164,6 +1158,7 @@ cc_library(
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:matrix_data_cc_proto",
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
@@ -1238,6 +1233,7 @@ cc_library(
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
|
||||
@@ -17,10 +17,13 @@
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
@@ -60,4 +63,22 @@ REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
|
||||
typedef BeginLoopCalculator<std::vector<Tensor>> BeginLoopTensorCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopTensorCalculator);
|
||||
|
||||
// A calculator to process std::vector<mediapipe::ImageFrame>.
|
||||
typedef BeginLoopCalculator<std::vector<ImageFrame>>
|
||||
BeginLoopImageFrameCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopImageFrameCalculator);
|
||||
|
||||
// A calculator to process std::vector<mediapipe::GpuBuffer>.
|
||||
typedef BeginLoopCalculator<std::vector<GpuBuffer>>
|
||||
BeginLoopGpuBufferCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator);
|
||||
|
||||
// A calculator to process std::vector<mediapipe::Image>.
|
||||
typedef BeginLoopCalculator<std::vector<Image>> BeginLoopImageCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopImageCalculator);
|
||||
|
||||
// A calculator to process std::vector<float>.
|
||||
typedef BeginLoopCalculator<std::vector<float>> BeginLoopFloatCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopFloatCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -15,47 +15,57 @@
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Calculator for implementing loops on iterable collections inside a MediaPipe
|
||||
// graph.
|
||||
// graph. Assume InputIterT is an iterable for type InputT, and OutputIterT is
|
||||
// an iterable for type OutputT, e.g. vector<InputT> and vector<OutputT>.
|
||||
// First, instantiate specializations in the loop calculators' implementations
|
||||
// if missing:
|
||||
// BeginLoopInputTCalculator = BeginLoopCalculator<InputIterT>
|
||||
// EndLoopOutputTCalculator = EndLoopCalculator<OutputIterT>
|
||||
// Then, the following graph transforms an item of type InputIterT to an
|
||||
// OutputIterT by applying InputToOutputConverter to every element:
|
||||
//
|
||||
// It is designed to be used like:
|
||||
//
|
||||
// node {
|
||||
// calculator: "BeginLoopWithIterableCalculator"
|
||||
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
|
||||
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
|
||||
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// node { # Type @timestamp
|
||||
// calculator: "BeginLoopInputTCalculator"
|
||||
// input_stream: "ITERABLE:input_iterable" # InputIterT @iterable_ts
|
||||
// input_stream: "CLONE:extra_input" # ExtraT @extra_ts
|
||||
// output_stream: "ITEM:input_iterator" # InputT @loop_internal_ts
|
||||
// output_stream: "CLONE:cloned_extra_input" # ExtraT @loop_internal_ts
|
||||
// output_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "ElementToBlaConverterSubgraph"
|
||||
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
|
||||
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// calculator: "InputToOutputConverter"
|
||||
// input_stream: "INPUT:input_iterator" # InputT @loop_internal_ts
|
||||
// input_stream: "EXTRA:cloned_extra_input" # ExtraT @loop_internal_ts
|
||||
// output_stream: "OUTPUT:output_iterator" # OutputT @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "EndLoopWithOutputCalculator"
|
||||
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
|
||||
// calculator: "EndLoopOutputTCalculator"
|
||||
// input_stream: "ITEM:output_iterator" # OutputT @loop_internal_ts
|
||||
// input_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
|
||||
// output_stream: "ITERABLE:output_iterable" # OutputIterT @iterable_ts
|
||||
// }
|
||||
//
|
||||
// The resulting 'output_iterable' has the same timestamp as 'input_iterable'.
|
||||
// The output packets of this calculator are part of the loop body and have
|
||||
// loop-internal timestamps that are unrelated to the input iterator timestamp.
|
||||
//
|
||||
// Input streams tagged with "CLONE" are cloned to the corresponding output
|
||||
// streams at loop timestamps. This ensures that a MediaPipe graph or sub-graph
|
||||
// can run multiple times, once per element in the "ITERABLE" for each pakcet
|
||||
// clone of the packets in the "CLONE" input streams.
|
||||
// streams at loop-internal timestamps. This ensures that a MediaPipe graph or
|
||||
// sub-graph can run multiple times, once per element in the "ITERABLE" for each
|
||||
// packet clone of the packets in the "CLONE" input streams. Think of CLONEd
|
||||
// inputs as loop-wide constants.
|
||||
template <typename IterableT>
|
||||
class BeginLoopCalculator : public CalculatorBase {
|
||||
using ItemT = typename IterableT::value_type;
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
@@ -55,6 +56,10 @@ MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator);
|
||||
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<std::string>
|
||||
ConcatenateStringVectorCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(ConcatenateStringVectorCalculator);
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
|
||||
@@ -100,4 +105,7 @@ typedef ConcatenateVectorCalculator<mediapipe::RenderData>
|
||||
ConcatenateRenderDataVectorCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<mediapipe::Image>
|
||||
ConcatenateImageVectorCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(ConcatenateImageVectorCalculator);
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -30,13 +30,15 @@ namespace mediapipe {
|
||||
typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
|
||||
|
||||
void AddInputVector(int index, const std::vector<int>& input, int64_t timestamp,
|
||||
template <typename T>
|
||||
void AddInputVector(int index, const std::vector<T>& input, int64_t timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
runner->MutableInputs()->Index(index).packets.push_back(
|
||||
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
|
||||
MakePacket<std::vector<T>>(input).At(Timestamp(timestamp)));
|
||||
}
|
||||
|
||||
void AddInputVectors(const std::vector<std::vector<int>>& inputs,
|
||||
template <typename T>
|
||||
void AddInputVectors(const std::vector<std::vector<T>>& inputs,
|
||||
int64_t timestamp, CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
AddInputVector(i, inputs[i], timestamp, runner);
|
||||
@@ -382,6 +384,23 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
TEST(ConcatenateStringVectorCalculatorTest, OneTimestamp) {
|
||||
CalculatorRunner runner("ConcatenateStringVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<std::string>> inputs = {
|
||||
{"a", "b"}, {"c"}, {"d", "e", "f"}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<std::string> expected_vector = {"a", "b", "c", "d", "e", "f"};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<std::string>>());
|
||||
}
|
||||
|
||||
typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
|
||||
TestConcatenateUniqueIntPtrCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator);
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/matrix_data.pb.h"
|
||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
@@ -85,8 +86,12 @@ class ConstantSidePacketCalculator : public CalculatorBase {
|
||||
packet.Set<LandmarkList>();
|
||||
} else if (packet_options.has_double_value()) {
|
||||
packet.Set<double>();
|
||||
} else if (packet_options.has_matrix_data_value()) {
|
||||
packet.Set<MatrixData>();
|
||||
} else if (packet_options.has_time_series_header_value()) {
|
||||
packet.Set<TimeSeriesHeader>();
|
||||
} else if (packet_options.has_int64_value()) {
|
||||
packet.Set<int64_t>();
|
||||
} else {
|
||||
return absl::InvalidArgumentError(
|
||||
"None of supported values were specified in options.");
|
||||
@@ -121,9 +126,13 @@ class ConstantSidePacketCalculator : public CalculatorBase {
|
||||
MakePacket<LandmarkList>(packet_options.landmark_list_value()));
|
||||
} else if (packet_options.has_double_value()) {
|
||||
packet.Set(MakePacket<double>(packet_options.double_value()));
|
||||
} else if (packet_options.has_matrix_data_value()) {
|
||||
packet.Set(MakePacket<MatrixData>(packet_options.matrix_data_value()));
|
||||
} else if (packet_options.has_time_series_header_value()) {
|
||||
packet.Set(MakePacket<TimeSeriesHeader>(
|
||||
packet_options.time_series_header_value()));
|
||||
} else if (packet_options.has_int64_value()) {
|
||||
packet.Set(MakePacket<int64_t>(packet_options.int64_value()));
|
||||
} else {
|
||||
return absl::InvalidArgumentError(
|
||||
"None of supported values were specified in options.");
|
||||
|
||||
@@ -19,6 +19,7 @@ package mediapipe;
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
import "mediapipe/framework/formats/classification.proto";
|
||||
import "mediapipe/framework/formats/landmark.proto";
|
||||
import "mediapipe/framework/formats/matrix_data.proto";
|
||||
import "mediapipe/framework/formats/time_series_header.proto";
|
||||
|
||||
message ConstantSidePacketCalculatorOptions {
|
||||
@@ -29,14 +30,16 @@ message ConstantSidePacketCalculatorOptions {
|
||||
message ConstantSidePacket {
|
||||
oneof value {
|
||||
int32 int_value = 1;
|
||||
uint64 uint64_value = 5;
|
||||
int64 int64_value = 11;
|
||||
float float_value = 2;
|
||||
double double_value = 9;
|
||||
bool bool_value = 3;
|
||||
string string_value = 4;
|
||||
uint64 uint64_value = 5;
|
||||
ClassificationList classification_list_value = 6;
|
||||
LandmarkList landmark_list_value = 7;
|
||||
double double_value = 9;
|
||||
TimeSeriesHeader time_series_header_value = 10;
|
||||
MatrixData matrix_data_value = 12;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "absl/strings/string_view.h"
|
||||
@@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
|
||||
DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
|
||||
DoTestSingleSidePacket("{ bool_value: true }", true);
|
||||
DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
|
||||
DoTestSingleSidePacket<int64_t>("{ int64_value: 63 }", 63);
|
||||
}
|
||||
|
||||
TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
|
||||
|
||||
@@ -19,10 +19,12 @@
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "mediapipe/util/render_data.pb.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
@@ -68,8 +70,18 @@ REGISTER_CALCULATOR(EndLoopMatrixCalculator);
|
||||
typedef EndLoopCalculator<std::vector<Tensor>> EndLoopTensorCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopTensorCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<ImageFrame>> EndLoopImageFrameCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopImageFrameCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<GpuBuffer>> EndLoopGpuBufferCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopGpuBufferCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<::mediapipe::Image>>
|
||||
EndLoopImageCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopImageCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<std::array<float, 16>>>
|
||||
EndLoopAffineMatrixCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -17,13 +17,11 @@
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
@@ -33,27 +31,7 @@ namespace mediapipe {
|
||||
// from the "BATCH_END" tagged input stream, it emits the aggregated results
|
||||
// at the original timestamp contained in the "BATCH_END" input stream.
|
||||
//
|
||||
// It is designed to be used like:
|
||||
//
|
||||
// node {
|
||||
// calculator: "BeginLoopWithIterableCalculator"
|
||||
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
|
||||
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
|
||||
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "ElementToBlaConverterSubgraph"
|
||||
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
|
||||
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "EndLoopWithOutputCalculator"
|
||||
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
|
||||
// }
|
||||
// See BeginLoopCalculator for a usage example.
|
||||
template <typename IterableT>
|
||||
class EndLoopCalculator : public CalculatorBase {
|
||||
using ItemT = typename IterableT::value_type;
|
||||
@@ -79,7 +57,7 @@ class EndLoopCalculator : public CalculatorBase {
|
||||
}
|
||||
// Try to consume the item and move it into the collection. If the items
|
||||
// are not consumable, then try to copy them instead. If the items are
|
||||
// not copiable, then an error will be returned.
|
||||
// not copyable, then an error will be returned.
|
||||
auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
|
||||
if (item_ptr_or.ok()) {
|
||||
input_stream_collection_->push_back(std::move(*item_ptr_or.value()));
|
||||
|
||||
@@ -42,7 +42,7 @@ constexpr char kOptionsTag[] = "OPTIONS";
|
||||
//
|
||||
// Increasing `max_in_flight` to 2 or more can yield the better throughput
|
||||
// when the graph exhibits a high degree of pipeline parallelism. Decreasing
|
||||
// `max_in_flight` to 0 can yield a better average latency, but at the cost of
|
||||
// `max_in_queue` to 0 can yield a better average latency, but at the cost of
|
||||
// lower throughput (lower framerate) due to the time during which the graph
|
||||
// is idle awaiting the next input frame.
|
||||
//
|
||||
|
||||
@@ -26,19 +26,15 @@ constexpr char kStateChangeTag[] = "STATE_CHANGE";
|
||||
constexpr char kDisallowTag[] = "DISALLOW";
|
||||
constexpr char kAllowTag[] = "ALLOW";
|
||||
|
||||
enum GateState {
|
||||
GATE_UNINITIALIZED,
|
||||
GATE_ALLOW,
|
||||
GATE_DISALLOW,
|
||||
};
|
||||
|
||||
std::string ToString(GateState state) {
|
||||
std::string ToString(GateCalculatorOptions::GateState state) {
|
||||
switch (state) {
|
||||
case GATE_UNINITIALIZED:
|
||||
case GateCalculatorOptions::UNSPECIFIED:
|
||||
return "UNSPECIFIED";
|
||||
case GateCalculatorOptions::GATE_UNINITIALIZED:
|
||||
return "UNINITIALIZED";
|
||||
case GATE_ALLOW:
|
||||
case GateCalculatorOptions::GATE_ALLOW:
|
||||
return "ALLOW";
|
||||
case GATE_DISALLOW:
|
||||
case GateCalculatorOptions::GATE_DISALLOW:
|
||||
return "DISALLOW";
|
||||
}
|
||||
DLOG(FATAL) << "Unknown GateState";
|
||||
@@ -153,10 +149,12 @@ class GateCalculator : public CalculatorBase {
|
||||
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
num_data_streams_ = cc->Inputs().NumEntries("");
|
||||
last_gate_state_ = GATE_UNINITIALIZED;
|
||||
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
|
||||
|
||||
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
|
||||
last_gate_state_ = options.initial_gate_state();
|
||||
|
||||
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
|
||||
|
||||
empty_packets_as_allow_ = options.empty_packets_as_allow();
|
||||
|
||||
if (!use_side_packet_for_allow_disallow_ &&
|
||||
@@ -184,10 +182,12 @@ class GateCalculator : public CalculatorBase {
|
||||
allow = !cc->Inputs().Tag(kDisallowTag).Get<bool>();
|
||||
}
|
||||
}
|
||||
const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
|
||||
const GateCalculatorOptions::GateState new_gate_state =
|
||||
allow ? GateCalculatorOptions::GATE_ALLOW
|
||||
: GateCalculatorOptions::GATE_DISALLOW;
|
||||
|
||||
if (cc->Outputs().HasTag(kStateChangeTag)) {
|
||||
if (last_gate_state_ != GATE_UNINITIALIZED &&
|
||||
if (last_gate_state_ != GateCalculatorOptions::GATE_UNINITIALIZED &&
|
||||
last_gate_state_ != new_gate_state) {
|
||||
VLOG(2) << "State transition in " << cc->NodeName() << " @ "
|
||||
<< cc->InputTimestamp().Value() << " from "
|
||||
@@ -223,7 +223,8 @@ class GateCalculator : public CalculatorBase {
|
||||
}
|
||||
|
||||
private:
|
||||
GateState last_gate_state_ = GATE_UNINITIALIZED;
|
||||
GateCalculatorOptions::GateState last_gate_state_ =
|
||||
GateCalculatorOptions::GATE_UNINITIALIZED;
|
||||
int num_data_streams_;
|
||||
bool empty_packets_as_allow_;
|
||||
bool use_side_packet_for_allow_disallow_ = false;
|
||||
|
||||
@@ -31,4 +31,13 @@ message GateCalculatorOptions {
|
||||
// Whether to allow or disallow the input streams to pass when no
|
||||
// ALLOW/DISALLOW input or side input is specified.
|
||||
optional bool allow = 2 [default = false];
|
||||
|
||||
enum GateState {
|
||||
UNSPECIFIED = 0;
|
||||
GATE_UNINITIALIZED = 1;
|
||||
GATE_ALLOW = 2;
|
||||
GATE_DISALLOW = 3;
|
||||
}
|
||||
|
||||
optional GateState initial_gate_state = 3 [default = GATE_UNINITIALIZED];
|
||||
}
|
||||
|
||||
@@ -458,5 +458,29 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
|
||||
ASSERT_EQ(0, output.size());
|
||||
}
|
||||
|
||||
// Must detect allow value for first timestamp as a state change when the
|
||||
// initial state is set to GATE_DISALLOW.
|
||||
TEST_F(GateCalculatorTest, StateChangeTriggeredWithInitialGateStateOption) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
input_stream: "ALLOW:allow"
|
||||
output_stream: "test_output"
|
||||
output_stream: "STATE_CHANGE:state_change"
|
||||
options: {
|
||||
[mediapipe.GateCalculatorOptions.ext] {
|
||||
initial_gate_state: GATE_DISALLOW
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
constexpr int64_t kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, "ALLOW", true);
|
||||
|
||||
const std::vector<Packet>& output =
|
||||
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
||||
ASSERT_EQ(1, output.size());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
@@ -37,5 +38,12 @@ using GetDetectionVectorItemCalculator =
|
||||
GetVectorItemCalculator<mediapipe::Detection>;
|
||||
REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
|
||||
|
||||
using GetNormalizedRectVectorItemCalculator =
|
||||
GetVectorItemCalculator<NormalizedRect>;
|
||||
REGISTER_CALCULATOR(GetNormalizedRectVectorItemCalculator);
|
||||
|
||||
using GetRectVectorItemCalculator = GetVectorItemCalculator<Rect>;
|
||||
REGISTER_CALCULATOR(GetRectVectorItemCalculator);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -35,7 +35,7 @@ class MatrixToVectorCalculatorTest
|
||||
void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
|
||||
|
||||
void AppendInput(const std::vector<float>& column_major_data,
|
||||
int64 timestamp) {
|
||||
int64_t timestamp) {
|
||||
ASSERT_EQ(num_input_samples_ * num_input_channels_,
|
||||
column_major_data.size());
|
||||
Eigen::Map<const Matrix> data_map(&column_major_data[0],
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -51,9 +51,9 @@ class SimpleRunner : public CalculatorRunner {
|
||||
|
||||
virtual ~SimpleRunner() {}
|
||||
|
||||
void SetInput(const std::vector<int64>& timestamp_list) {
|
||||
void SetInput(const std::vector<int64_t>& timestamp_list) {
|
||||
MutableInputs()->Index(0).packets.clear();
|
||||
for (const int64 ts : timestamp_list) {
|
||||
for (const int64_t ts : timestamp_list) {
|
||||
MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(new std::string(absl::StrCat("Frame #", ts)))
|
||||
.At(Timestamp(ts)));
|
||||
@@ -72,8 +72,8 @@ class SimpleRunner : public CalculatorRunner {
|
||||
}
|
||||
|
||||
void CheckOutputTimestamps(
|
||||
const std::vector<int64>& expected_frames,
|
||||
const std::vector<int64>& expected_timestamps) const {
|
||||
const std::vector<int64_t>& expected_frames,
|
||||
const std::vector<int64_t>& expected_timestamps) const {
|
||||
EXPECT_EQ(expected_frames.size(), Outputs().Index(0).packets.size());
|
||||
EXPECT_EQ(expected_timestamps.size(), Outputs().Index(0).packets.size());
|
||||
int count = 0;
|
||||
@@ -112,7 +112,7 @@ MATCHER_P2(PacketAtTimestamp, payload, timestamp,
|
||||
*result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
|
||||
return false;
|
||||
}
|
||||
int64 actual_payload = arg.template Get<int64>();
|
||||
int64_t actual_payload = arg.template Get<int64_t>();
|
||||
if (actual_payload != payload) {
|
||||
*result_listener << "with incorrect payload = " << actual_payload;
|
||||
return false;
|
||||
@@ -137,18 +137,18 @@ class ReproducibleJitterWithReflectionStrategyForTesting
|
||||
//
|
||||
// An EXPECT will fail if sequence is less than the number requested during
|
||||
// processing.
|
||||
static std::vector<uint64> random_sequence;
|
||||
static std::vector<uint64_t> random_sequence;
|
||||
|
||||
protected:
|
||||
virtual uint64 GetNextRandom(uint64 n) {
|
||||
virtual uint64_t GetNextRandom(uint64_t n) {
|
||||
EXPECT_LT(sequence_index_, random_sequence.size());
|
||||
return random_sequence[sequence_index_++] % n;
|
||||
}
|
||||
|
||||
private:
|
||||
int32 sequence_index_ = 0;
|
||||
int32_t sequence_index_ = 0;
|
||||
};
|
||||
std::vector<uint64>
|
||||
std::vector<uint64_t>
|
||||
ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
|
||||
|
||||
// PacketResamplerCalculator child class which injects a specified stream
|
||||
@@ -469,7 +469,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
||||
}
|
||||
)pb"));
|
||||
|
||||
for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
|
||||
for (const int64_t ts : {0, 5000, 10010, 15001, 19990}) {
|
||||
runner.MutableInputs()->Tag(kDataTag).packets.push_back(
|
||||
Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts)));
|
||||
}
|
||||
|
||||
@@ -123,7 +123,10 @@ class PreviousLoopbackCalculator : public Node {
|
||||
// However, LOOP packet is empty.
|
||||
kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
|
||||
} else {
|
||||
kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
|
||||
// Avoids sending leftovers to a stream that's already closed.
|
||||
if (!kPrevLoop(cc).IsClosed()) {
|
||||
kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
|
||||
}
|
||||
}
|
||||
loop_packets_.pop_front();
|
||||
main_packet_specs_.pop_front();
|
||||
|
||||
@@ -43,8 +43,8 @@ constexpr char kDisallowTag[] = "DISALLOW";
|
||||
|
||||
// Returns the timestamp values for a vector of Packets.
|
||||
// TODO: puth this kind of test util in a common place.
|
||||
std::vector<int64> TimestampValues(const std::vector<Packet>& packets) {
|
||||
std::vector<int64> result;
|
||||
std::vector<int64_t> TimestampValues(const std::vector<Packet>& packets) {
|
||||
std::vector<int64_t> result;
|
||||
for (const Packet& packet : packets) {
|
||||
result.push_back(packet.Timestamp().Value());
|
||||
}
|
||||
@@ -371,7 +371,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
|
||||
for (int main_ts = 0; main_ts < 50; ++main_ts) {
|
||||
send_packet("in", main_ts);
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
std::vector<int64> ts_values = TimestampValues(outputs);
|
||||
std::vector<int64_t> ts_values = TimestampValues(outputs);
|
||||
EXPECT_EQ(ts_values.size(), main_ts + 1);
|
||||
for (int j = 0; j < main_ts + 1; ++j) {
|
||||
EXPECT_EQ(ts_values[j], j);
|
||||
|
||||
@@ -121,7 +121,7 @@ absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
|
||||
if (cc->Outputs().HasTag(kTagAtTimestamp)) {
|
||||
RET_CHECK_EQ(num_entries + 1, cc->InputSidePackets().NumEntries())
|
||||
<< "For AT_TIMESTAMP tag, 2 input side packets are required.";
|
||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set<int64>();
|
||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set<int64_t>();
|
||||
} else {
|
||||
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
|
||||
<< "Same number of input side packets and output streams is required.";
|
||||
@@ -178,8 +178,8 @@ absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
|
||||
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
||||
}
|
||||
} else if (cc->Outputs().HasTag(kTagAtTimestamp)) {
|
||||
int64 timestamp =
|
||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get<int64>();
|
||||
int64_t timestamp =
|
||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get<int64_t>();
|
||||
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
||||
cc->Outputs()
|
||||
.Get(output_tag_, i)
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
@@ -86,4 +87,12 @@ REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
|
||||
typedef SplitVectorCalculator<float, false> SplitFloatVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitFloatVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<mediapipe::Image, false>
|
||||
SplitImageVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitImageVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<std::array<float, 16>, false>
|
||||
SplitAffineMatrixVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitAffineMatrixVectorCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -12,11 +12,13 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// A calculator that takes a packet of an input stream and converts it to an
|
||||
// output side packet. This calculator only works under the assumption that the
|
||||
@@ -28,21 +30,21 @@ namespace mediapipe {
|
||||
// input_stream: "stream"
|
||||
// output_side_packet: "side_packet"
|
||||
// }
|
||||
class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
|
||||
class StreamToSidePacketCalculator : public Node {
|
||||
public:
|
||||
static absl::Status GetContract(mediapipe::CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
cc->OutputSidePackets().Index(0).SetAny();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
static constexpr Input<AnyType>::Optional kIn{""};
|
||||
static constexpr SideOutput<SameType<kIn>> kOut{""};
|
||||
|
||||
MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
|
||||
|
||||
absl::Status Process(mediapipe::CalculatorContext* cc) override {
|
||||
mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
|
||||
cc->OutputSidePackets().Index(0).Set(
|
||||
packet.At(mediapipe::Timestamp::Unset()));
|
||||
kOut(cc).Set(
|
||||
kIn(cc).packet().As<AnyType>().At(mediapipe::Timestamp::Unset()));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(StreamToSidePacketCalculator);
|
||||
|
||||
MEDIAPIPE_REGISTER_NODE(StreamToSidePacketCalculator);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -135,7 +135,6 @@ cc_library(
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
@@ -317,6 +316,7 @@ cc_library(
|
||||
cc_test(
|
||||
name = "image_cropping_calculator_test",
|
||||
srcs = ["image_cropping_calculator_test.cc"],
|
||||
tags = ["not_run:arm"],
|
||||
deps = [
|
||||
":image_cropping_calculator",
|
||||
":image_cropping_calculator_cc_proto",
|
||||
@@ -650,6 +650,7 @@ cc_library(
|
||||
cc_test(
|
||||
name = "segmentation_smoothing_calculator_test",
|
||||
srcs = ["segmentation_smoothing_calculator_test.cc"],
|
||||
tags = ["not_run:arm"],
|
||||
deps = [
|
||||
":image_clone_calculator",
|
||||
":image_clone_calculator_cc_proto",
|
||||
@@ -771,7 +772,10 @@ cc_test(
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero_interp_cubic.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
|
||||
],
|
||||
tags = ["desktop_only_test"],
|
||||
tags = [
|
||||
"desktop_only_test",
|
||||
"not_run:arm",
|
||||
],
|
||||
deps = [
|
||||
":affine_transformation",
|
||||
":image_transformation_calculator",
|
||||
|
||||
@@ -112,7 +112,7 @@ class BilateralFilterCalculator : public CalculatorBase {
|
||||
REGISTER_CALCULATOR(BilateralFilterCalculator);
|
||||
|
||||
absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
|
||||
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||
|
||||
if (cc->Inputs().HasTag(kInputFrameTag) &&
|
||||
cc->Inputs().HasTag(kInputFrameTagGpu)) {
|
||||
|
||||
@@ -110,7 +110,7 @@ REGISTER_CALCULATOR(SegmentationSmoothingCalculator);
|
||||
|
||||
absl::Status SegmentationSmoothingCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||
|
||||
cc->Inputs().Tag(kCurrentMaskTag).Set<Image>();
|
||||
cc->Inputs().Tag(kPreviousMaskTag).Set<Image>();
|
||||
|
||||
@@ -142,7 +142,7 @@ class SetAlphaCalculator : public CalculatorBase {
|
||||
REGISTER_CALCULATOR(SetAlphaCalculator);
|
||||
|
||||
absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
|
||||
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||
|
||||
bool use_gpu = false;
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
|
||||
buf[0] = (fourcc >> 24) & 0xff;
|
||||
buf[1] = (fourcc >> 16) & 0xff;
|
||||
buf[2] = (fourcc >> 8) & 0xff;
|
||||
buf[3] = (fourcc)&0xff;
|
||||
buf[3] = (fourcc) & 0xff;
|
||||
buf[4] = 0;
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
@@ -228,7 +228,6 @@ cc_library(
|
||||
"//mediapipe/tasks/metadata:metadata_schema_cc",
|
||||
"@com_google_absl//absl/container:flat_hash_set",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
alwayslink = 1,
|
||||
@@ -280,7 +279,6 @@ cc_library(
|
||||
"//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
|
||||
"//mediapipe/tasks/metadata:metadata_schema_cc",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -394,7 +392,7 @@ mediapipe_proto_library(
|
||||
# If you want to have precise control of which implementations to include (e.g. for strict binary
|
||||
# size concerns), depend on those implementations directly, and do not depend on
|
||||
# :inference_calculator.
|
||||
# In all cases, use "InferenceCalulator" in your graphs.
|
||||
# In all cases, use "InferenceCalculator" in your graphs.
|
||||
cc_library_with_tflite(
|
||||
name = "inference_calculator_interface",
|
||||
srcs = ["inference_calculator.cc"],
|
||||
@@ -655,6 +653,11 @@ cc_library(
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
|
||||
}) + select({
|
||||
"//mediapipe:apple": [
|
||||
"//third_party/apple_frameworks:MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -1052,6 +1055,7 @@ cc_test(
|
||||
"testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
|
||||
"testdata/image_to_tensor/noop_except_range.png",
|
||||
],
|
||||
tags = ["not_run:arm"],
|
||||
deps = [
|
||||
":image_to_tensor_calculator",
|
||||
":image_to_tensor_converter",
|
||||
|
||||
@@ -282,18 +282,23 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
|
||||
if (options.has_volume_gain_db()) {
|
||||
gain_ = pow(10, options.volume_gain_db() / 20.0);
|
||||
}
|
||||
RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
|
||||
!kAudioIn(cc).Header().IsEmpty())
|
||||
<< "Must either specify the time series header of the \"AUDIO\" stream "
|
||||
"or have the \"SAMPLE_RATE\" stream connected.";
|
||||
if (!kAudioIn(cc).Header().IsEmpty()) {
|
||||
mediapipe::TimeSeriesHeader input_header;
|
||||
MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
|
||||
kAudioIn(cc).Header(), &input_header));
|
||||
if (stream_mode_) {
|
||||
MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
|
||||
} else {
|
||||
source_sample_rate_ = input_header.sample_rate();
|
||||
if (options.has_source_sample_rate()) {
|
||||
source_sample_rate_ = options.source_sample_rate();
|
||||
} else {
|
||||
RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
|
||||
!kAudioIn(cc).Header().IsEmpty())
|
||||
<< "Must either specify the time series header of the \"AUDIO\" stream "
|
||||
"or have the \"SAMPLE_RATE\" stream connected.";
|
||||
if (!kAudioIn(cc).Header().IsEmpty()) {
|
||||
mediapipe::TimeSeriesHeader input_header;
|
||||
MP_RETURN_IF_ERROR(
|
||||
mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
|
||||
kAudioIn(cc).Header(), &input_header));
|
||||
if (stream_mode_) {
|
||||
MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
|
||||
} else {
|
||||
source_sample_rate_ = input_header.sample_rate();
|
||||
}
|
||||
}
|
||||
}
|
||||
AppendZerosToSampleBuffer(padding_samples_before_);
|
||||
|
||||
@@ -85,4 +85,7 @@ message AudioToTensorCalculatorOptions {
|
||||
// The volume gain, measured in dB.
|
||||
// Scale the input audio amplitude by 10^(volume_gain_db/20).
|
||||
optional double volume_gain_db = 12;
|
||||
|
||||
// The source number of samples per second (hertz) of the input audio buffers.
|
||||
optional double source_sample_rate = 13;
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "absl/container/flat_hash_set.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "absl/strings/ascii.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
@@ -244,7 +243,8 @@ std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
|
||||
input_tensors.reserve(kNumInputTensorsForBert);
|
||||
for (int i = 0; i < kNumInputTensorsForBert; ++i) {
|
||||
input_tensors.push_back(
|
||||
{Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})});
|
||||
{Tensor::ElementType::kInt32,
|
||||
Tensor::Shape({1, tensor_size}, has_dynamic_input_tensors_)});
|
||||
}
|
||||
std::memcpy(input_tensors[input_ids_tensor_index_]
|
||||
.GetCpuWriteView()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -95,7 +95,8 @@ absl::Status FrameBufferProcessor::Convert(const mediapipe::Image& input,
|
||||
static_cast<int>(range_max) == 255);
|
||||
}
|
||||
|
||||
auto input_frame = input.GetGpuBuffer().GetReadView<FrameBuffer>();
|
||||
auto input_frame =
|
||||
input.GetGpuBuffer(/*upload_to_gpu=*/false).GetReadView<FrameBuffer>();
|
||||
const auto& output_shape = output_tensor.shape();
|
||||
MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
|
||||
FrameBuffer::Dimension output_dimension{/*width=*/output_shape.dims[2],
|
||||
|
||||
@@ -69,6 +69,7 @@ class InferenceCalculatorGlAdvancedImpl
|
||||
gpu_delegate_options);
|
||||
absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
|
||||
absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
|
||||
bool UseSerializedModel() const { return use_serialized_model_; }
|
||||
|
||||
private:
|
||||
bool use_kernel_caching_ = false;
|
||||
@@ -150,8 +151,6 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
|
||||
}
|
||||
|
||||
absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() {
|
||||
MP_RETURN_IF_ERROR(
|
||||
on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get()));
|
||||
return gpu_helper_.RunInGlContext([this]() -> absl::Status {
|
||||
tflite_gpu_runner_.reset();
|
||||
return absl::OkStatus();
|
||||
@@ -226,9 +225,14 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
|
||||
tflite_gpu_runner_->GetOutputShapes()[i].c};
|
||||
}
|
||||
|
||||
if (on_disk_cache_helper_.UseSerializedModel()) {
|
||||
tflite_gpu_runner_->ForceOpenCLInitFromSerializedModel();
|
||||
}
|
||||
|
||||
MP_RETURN_IF_ERROR(
|
||||
on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get()));
|
||||
return tflite_gpu_runner_->Build();
|
||||
MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build());
|
||||
return on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get());
|
||||
}
|
||||
|
||||
#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
|
||||
|
||||
@@ -96,6 +96,19 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
|
||||
CalculatorContext* cc, const std::vector<Tensor>& input_tensors) {
|
||||
// Read CPU input into tensors.
|
||||
RET_CHECK_EQ(interpreter_->inputs().size(), input_tensors.size());
|
||||
|
||||
// If the input tensors have dynamic shape, then the tensors need to be
|
||||
// resized and reallocated before we can copy the tensor values.
|
||||
bool resized_tensor_shapes = false;
|
||||
for (int i = 0; i < input_tensors.size(); ++i) {
|
||||
if (input_tensors[i].shape().is_dynamic) {
|
||||
interpreter_->ResizeInputTensorStrict(i, input_tensors[i].shape().dims);
|
||||
resized_tensor_shapes = true;
|
||||
}
|
||||
}
|
||||
// Reallocation is needed for memory sanity.
|
||||
if (resized_tensor_shapes) interpreter_->AllocateTensors();
|
||||
|
||||
for (int i = 0; i < input_tensors.size(); ++i) {
|
||||
const TfLiteType input_tensor_type =
|
||||
interpreter_->tensor(interpreter_->inputs()[i])->type;
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/tensor/regex_preprocessor_calculator.pb.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/api2/port.h"
|
||||
@@ -161,7 +160,7 @@ absl::Status RegexPreprocessorCalculator::Process(CalculatorContext* cc) {
|
||||
// not found in the tokenizer vocab.
|
||||
std::vector<Tensor> result;
|
||||
result.push_back(
|
||||
{Tensor::ElementType::kInt32, Tensor::Shape({max_seq_len_})});
|
||||
{Tensor::ElementType::kInt32, Tensor::Shape({1, max_seq_len_})});
|
||||
std::memcpy(result[0].GetCpuWriteView().buffer<int32_t>(),
|
||||
input_tokens.data(), input_tokens.size() * sizeof(int32_t));
|
||||
kTensorsOut(cc).Send(std::move(result));
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -256,6 +256,7 @@ class TensorsToDetectionsCalculator : public Node {
|
||||
|
||||
bool gpu_inited_ = false;
|
||||
bool gpu_input_ = false;
|
||||
bool gpu_has_enough_work_groups_ = true;
|
||||
bool anchors_init_ = false;
|
||||
};
|
||||
MEDIAPIPE_REGISTER_NODE(TensorsToDetectionsCalculator);
|
||||
@@ -291,7 +292,7 @@ absl::Status TensorsToDetectionsCalculator::Open(CalculatorContext* cc) {
|
||||
absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
|
||||
auto output_detections = absl::make_unique<std::vector<Detection>>();
|
||||
bool gpu_processing = false;
|
||||
if (CanUseGpu()) {
|
||||
if (CanUseGpu() && gpu_has_enough_work_groups_) {
|
||||
// Use GPU processing only if at least one input tensor is already on GPU
|
||||
// (to avoid CPU->GPU overhead).
|
||||
for (const auto& tensor : *kInTensors(cc)) {
|
||||
@@ -321,11 +322,20 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
|
||||
RET_CHECK(!has_custom_box_indices_);
|
||||
}
|
||||
|
||||
if (gpu_processing) {
|
||||
if (!gpu_inited_) {
|
||||
MP_RETURN_IF_ERROR(GpuInit(cc));
|
||||
if (gpu_processing && !gpu_inited_) {
|
||||
auto status = GpuInit(cc);
|
||||
if (status.ok()) {
|
||||
gpu_inited_ = true;
|
||||
} else if (status.code() == absl::StatusCode::kFailedPrecondition) {
|
||||
// For initialization error because of hardware limitation, fallback to
|
||||
// CPU processing.
|
||||
LOG(WARNING) << status.message();
|
||||
} else {
|
||||
// For other error, let the error propagates.
|
||||
return status;
|
||||
}
|
||||
}
|
||||
if (gpu_processing && gpu_inited_) {
|
||||
MP_RETURN_IF_ERROR(ProcessGPU(cc, output_detections.get()));
|
||||
} else {
|
||||
MP_RETURN_IF_ERROR(ProcessCPU(cc, output_detections.get()));
|
||||
@@ -346,17 +356,41 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
|
||||
// TODO: Add flexible input tensor size handling.
|
||||
auto raw_box_tensor =
|
||||
&input_tensors[tensor_mapping_.detections_tensor_index()];
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims.size(), 3);
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
|
||||
RET_CHECK_GT(num_boxes_, 0) << "Please set num_boxes in calculator options";
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
|
||||
if (raw_box_tensor->shape().dims.size() == 3) {
|
||||
// The tensors from CPU inference has dim 3.
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
|
||||
} else if (raw_box_tensor->shape().dims.size() == 4) {
|
||||
// The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
|
||||
// we allow tensors with 4 dims.
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[1], 1);
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_boxes_);
|
||||
RET_CHECK_EQ(raw_box_tensor->shape().dims[3], num_coords_);
|
||||
} else {
|
||||
return absl::InvalidArgumentError(
|
||||
"The dimensions of box Tensor must be 3 or 4.");
|
||||
}
|
||||
auto raw_score_tensor =
|
||||
&input_tensors[tensor_mapping_.scores_tensor_index()];
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims.size(), 3);
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
|
||||
if (raw_score_tensor->shape().dims.size() == 3) {
|
||||
// The tensors from CPU inference has dim 3.
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
|
||||
} else if (raw_score_tensor->shape().dims.size() == 4) {
|
||||
// The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
|
||||
// we allow tensors with 4 dims.
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[1], 1);
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_boxes_);
|
||||
RET_CHECK_EQ(raw_score_tensor->shape().dims[3], num_classes_);
|
||||
} else {
|
||||
return absl::InvalidArgumentError(
|
||||
"The dimensions of score Tensor must be 3 or 4.");
|
||||
}
|
||||
auto raw_box_view = raw_box_tensor->GetCpuReadView();
|
||||
auto raw_boxes = raw_box_view.buffer<float>();
|
||||
auto raw_scores_view = raw_score_tensor->GetCpuReadView();
|
||||
@@ -1111,8 +1145,13 @@ void main() {
|
||||
int max_wg_size; // typically <= 1024
|
||||
glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1,
|
||||
&max_wg_size); // y-dim
|
||||
CHECK_LT(num_classes_, max_wg_size)
|
||||
<< "# classes must be < " << max_wg_size;
|
||||
gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
|
||||
if (!gpu_has_enough_work_groups_) {
|
||||
return absl::FailedPreconditionError(absl::StrFormat(
|
||||
"Hardware limitation: Processing will be done on CPU, because "
|
||||
"num_classes %d exceeds the max work_group size %d.",
|
||||
num_classes_, max_wg_size));
|
||||
}
|
||||
// TODO support better filtering.
|
||||
if (class_index_set_.is_allowlist) {
|
||||
CHECK_EQ(class_index_set_.values.size(),
|
||||
@@ -1370,7 +1409,13 @@ kernel void scoreKernel(
|
||||
Tensor::ElementType::kFloat32, Tensor::Shape{1, num_boxes_ * 2});
|
||||
// # filter classes supported is hardware dependent.
|
||||
int max_wg_size = score_program_.maxTotalThreadsPerThreadgroup;
|
||||
CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size;
|
||||
gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
|
||||
if (!gpu_has_enough_work_groups_) {
|
||||
return absl::FailedPreconditionError(absl::StrFormat(
|
||||
"Hardware limitation: Processing will be done on CPU, because "
|
||||
"num_classes %d exceeds the max work_group size %d.",
|
||||
num_classes_, max_wg_size));
|
||||
}
|
||||
}
|
||||
|
||||
#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
|
||||
@@ -400,6 +400,21 @@ cc_library(
|
||||
# compile your binary with the flag TENSORFLOW_PROTOS=lite.
|
||||
cc_library(
|
||||
name = "tensorflow_inference_calculator_no_envelope_loader",
|
||||
deps = [
|
||||
":tensorflow_inference_calculator_for_boq",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
# This dependency removed the following 3 targets because they failed Boq conformance test:
|
||||
#
|
||||
# tensorflow_jellyfish_deps
|
||||
# jfprof_lib
|
||||
# xprofilez_with_server
|
||||
#
|
||||
# If you need them plz consider tensorflow_inference_calculator_no_envelope_loader.
|
||||
cc_library(
|
||||
name = "tensorflow_inference_calculator_for_boq",
|
||||
srcs = ["tensorflow_inference_calculator.cc"],
|
||||
deps = [
|
||||
":tensorflow_inference_calculator_cc_proto",
|
||||
@@ -585,6 +600,24 @@ cc_library(
|
||||
# See yaqs/1092546221614039040
|
||||
cc_library(
|
||||
name = "tensorflow_session_from_saved_model_generator_no_envelope_loader",
|
||||
defines = select({
|
||||
"//mediapipe:android": ["__ANDROID__"],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
deps = [
|
||||
":tensorflow_session_from_saved_model_generator_for_boq",
|
||||
] + select({
|
||||
"//conditions:default": [
|
||||
"//learning/brain/frameworks/uptc/public:uptc_session_no_envelope_loader",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
# Same library as tensorflow_session_from_saved_model_generator without uptc_session,
|
||||
# envelop_loader and remote_session dependencies.
|
||||
cc_library(
|
||||
name = "tensorflow_session_from_saved_model_generator_for_boq",
|
||||
srcs = ["tensorflow_session_from_saved_model_generator.cc"],
|
||||
defines = select({
|
||||
"//mediapipe:android": ["__ANDROID__"],
|
||||
@@ -899,7 +932,6 @@ cc_test(
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:location",
|
||||
"//mediapipe/framework/formats:location_opencv",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
@@ -1049,6 +1081,7 @@ cc_test(
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":tensor_to_image_frame_calculator",
|
||||
":tensor_to_image_frame_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
|
||||
@@ -164,8 +164,8 @@ class PackMediaSequenceCalculator : public CalculatorBase {
|
||||
}
|
||||
}
|
||||
|
||||
CHECK(cc->Outputs().HasTag(kSequenceExampleTag) ||
|
||||
cc->OutputSidePackets().HasTag(kSequenceExampleTag))
|
||||
RET_CHECK(cc->Outputs().HasTag(kSequenceExampleTag) ||
|
||||
cc->OutputSidePackets().HasTag(kSequenceExampleTag))
|
||||
<< "Neither the output stream nor the output side packet is set to "
|
||||
"output the sequence example.";
|
||||
if (cc->Outputs().HasTag(kSequenceExampleTag)) {
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/location.h"
|
||||
#include "mediapipe/framework/formats/location_opencv.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
@@ -96,7 +95,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImages) {
|
||||
mpms::SetClipMediaId(test_video_id, input_sequence.get());
|
||||
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
|
||||
ASSERT_TRUE(
|
||||
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
|
||||
OpenCvImageEncoderCalculatorResults encoded_image;
|
||||
encoded_image.set_encoded_image(bytes.data(), bytes.size());
|
||||
encoded_image.set_width(2);
|
||||
@@ -139,7 +139,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoPrefixedImages) {
|
||||
mpms::SetClipMediaId(test_video_id, input_sequence.get());
|
||||
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
|
||||
ASSERT_TRUE(
|
||||
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
|
||||
OpenCvImageEncoderCalculatorResults encoded_image;
|
||||
encoded_image.set_encoded_image(bytes.data(), bytes.size());
|
||||
encoded_image.set_width(2);
|
||||
@@ -378,7 +379,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
|
||||
Adopt(input_sequence.release());
|
||||
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
|
||||
ASSERT_TRUE(
|
||||
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
|
||||
OpenCvImageEncoderCalculatorResults encoded_image;
|
||||
encoded_image.set_encoded_image(bytes.data(), bytes.size());
|
||||
auto image_ptr =
|
||||
@@ -410,7 +412,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoForwardFlowEncodeds) {
|
||||
|
||||
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
|
||||
ASSERT_TRUE(
|
||||
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
|
||||
std::string test_flow_string(bytes.begin(), bytes.end());
|
||||
OpenCvImageEncoderCalculatorResults encoded_flow;
|
||||
encoded_flow.set_encoded_image(test_flow_string);
|
||||
@@ -618,7 +621,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
|
||||
}
|
||||
cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
|
||||
ASSERT_TRUE(
|
||||
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
|
||||
OpenCvImageEncoderCalculatorResults encoded_image;
|
||||
encoded_image.set_encoded_image(bytes.data(), bytes.size());
|
||||
encoded_image.set_width(width);
|
||||
@@ -767,7 +771,8 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) {
|
||||
|
||||
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
|
||||
ASSERT_TRUE(
|
||||
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
|
||||
std::string test_flow_string(bytes.begin(), bytes.end());
|
||||
OpenCvImageEncoderCalculatorResults encoded_flow;
|
||||
encoded_flow.set_encoded_image(test_flow_string);
|
||||
@@ -813,7 +818,8 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamNotOK) {
|
||||
mpms::SetClipMediaId(test_video_id, input_sequence.get());
|
||||
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
|
||||
ASSERT_TRUE(
|
||||
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
|
||||
std::string test_flow_string(bytes.begin(), bytes.end());
|
||||
OpenCvImageEncoderCalculatorResults encoded_flow;
|
||||
encoded_flow.set_encoded_image(test_flow_string);
|
||||
@@ -970,7 +976,8 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
|
||||
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
|
||||
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
|
||||
ASSERT_TRUE(
|
||||
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
|
||||
OpenCvImageEncoderCalculatorResults encoded_image;
|
||||
encoded_image.set_encoded_image(bytes.data(), bytes.size());
|
||||
encoded_image.set_width(2);
|
||||
@@ -1021,7 +1028,8 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
|
||||
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
|
||||
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
|
||||
std::vector<uchar> bytes;
|
||||
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
|
||||
ASSERT_TRUE(
|
||||
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
|
||||
OpenCvImageEncoderCalculatorResults encoded_image;
|
||||
encoded_image.set_encoded_image(bytes.data(), bytes.size());
|
||||
int height = 2;
|
||||
|
||||
@@ -65,6 +65,7 @@ class TensorToImageFrameCalculator : public CalculatorBase {
|
||||
|
||||
private:
|
||||
float scale_factor_;
|
||||
bool scale_per_frame_min_max_;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(TensorToImageFrameCalculator);
|
||||
@@ -88,6 +89,8 @@ absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) {
|
||||
absl::Status TensorToImageFrameCalculator::Open(CalculatorContext* cc) {
|
||||
scale_factor_ =
|
||||
cc->Options<TensorToImageFrameCalculatorOptions>().scale_factor();
|
||||
scale_per_frame_min_max_ = cc->Options<TensorToImageFrameCalculatorOptions>()
|
||||
.scale_per_frame_min_max();
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
@@ -109,16 +112,38 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) {
|
||||
auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8);
|
||||
const int32_t total_size = height * width * depth;
|
||||
|
||||
if (scale_per_frame_min_max_) {
|
||||
RET_CHECK_EQ(input_tensor.dtype(), tensorflow::DT_FLOAT)
|
||||
<< "Setting scale_per_frame_min_max requires FLOAT input tensors.";
|
||||
}
|
||||
::std::unique_ptr<const ImageFrame> output;
|
||||
if (input_tensor.dtype() == tensorflow::DT_FLOAT) {
|
||||
// Allocate buffer with alignments.
|
||||
std::unique_ptr<uint8_t[]> buffer(
|
||||
new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]);
|
||||
auto data = input_tensor.flat<float>().data();
|
||||
float min = 1e23;
|
||||
float max = -1e23;
|
||||
if (scale_per_frame_min_max_) {
|
||||
for (int i = 0; i < total_size; ++i) {
|
||||
float d = scale_factor_ * data[i];
|
||||
if (d < min) {
|
||||
min = d;
|
||||
}
|
||||
if (d > max) {
|
||||
max = d;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < total_size; ++i) {
|
||||
float d = scale_factor_ * data[i];
|
||||
if (d < 0) d = 0;
|
||||
if (d > 255) d = 255;
|
||||
float d = data[i];
|
||||
if (scale_per_frame_min_max_) {
|
||||
d = 255 * (d - min) / (max - min + 1e-9);
|
||||
} else {
|
||||
d = scale_factor_ * d;
|
||||
if (d < 0) d = 0;
|
||||
if (d > 255) d = 255;
|
||||
}
|
||||
buffer[i] = d;
|
||||
}
|
||||
output = ::absl::make_unique<ImageFrame>(
|
||||
|
||||
@@ -26,4 +26,8 @@ message TensorToImageFrameCalculatorOptions {
|
||||
// Multiples floating point tensor outputs by this value before converting to
|
||||
// uint8. This is useful for converting from range [0, 1] to [0, 255]
|
||||
optional float scale_factor = 1 [default = 1.0];
|
||||
|
||||
// If true, scales any FLOAT tensor input of [min, max] to be between [0, 255]
|
||||
// per frame. This overrides any explicit scale_factor.
|
||||
optional bool scale_per_frame_min_max = 2 [default = false];
|
||||
}
|
||||
|
||||
@@ -11,7 +11,9 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include <type_traits>
|
||||
|
||||
#include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
@@ -32,11 +34,14 @@ constexpr char kImage[] = "IMAGE";
|
||||
template <class TypeParam>
|
||||
class TensorToImageFrameCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUpRunner() {
|
||||
void SetUpRunner(bool scale_per_frame_min_max = false) {
|
||||
CalculatorGraphConfig::Node config;
|
||||
config.set_calculator("TensorToImageFrameCalculator");
|
||||
config.add_input_stream("TENSOR:input_tensor");
|
||||
config.add_output_stream("IMAGE:output_image");
|
||||
config.mutable_options()
|
||||
->MutableExtension(mediapipe::TensorToImageFrameCalculatorOptions::ext)
|
||||
->set_scale_per_frame_min_max(scale_per_frame_min_max);
|
||||
runner_ = absl::make_unique<CalculatorRunner>(config);
|
||||
}
|
||||
|
||||
@@ -157,4 +162,47 @@ TYPED_TEST(TensorToImageFrameCalculatorTest,
|
||||
}
|
||||
}
|
||||
|
||||
TYPED_TEST(TensorToImageFrameCalculatorTest,
|
||||
Converts3DTensorToImageFrame2DGrayWithScaling) {
|
||||
this->SetUpRunner(true);
|
||||
auto& runner = this->runner_;
|
||||
constexpr int kWidth = 16;
|
||||
constexpr int kHeight = 8;
|
||||
const tf::TensorShape tensor_shape{kHeight, kWidth};
|
||||
auto tensor = absl::make_unique<tf::Tensor>(
|
||||
tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
|
||||
auto tensor_vec = tensor->template flat<TypeParam>().data();
|
||||
|
||||
// Writing sequence of integers as floats which we want normalized.
|
||||
tensor_vec[0] = 255;
|
||||
for (int i = 1; i < kWidth * kHeight; ++i) {
|
||||
tensor_vec[i] = 200;
|
||||
}
|
||||
|
||||
const int64_t time = 1234;
|
||||
runner->MutableInputs()->Tag(kTensor).packets.push_back(
|
||||
Adopt(tensor.release()).At(Timestamp(time)));
|
||||
|
||||
if (!std::is_same<TypeParam, float>::value) {
|
||||
EXPECT_FALSE(runner->Run().ok());
|
||||
return; // Short circuit because does not apply to other types.
|
||||
} else {
|
||||
EXPECT_TRUE(runner->Run().ok());
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner->Outputs().Tag(kImage).packets;
|
||||
EXPECT_EQ(1, output_packets.size());
|
||||
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
|
||||
const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
|
||||
EXPECT_EQ(ImageFormat::GRAY8, output_image.Format());
|
||||
EXPECT_EQ(kWidth, output_image.Width());
|
||||
EXPECT_EQ(kHeight, output_image.Height());
|
||||
|
||||
EXPECT_EQ(255, output_image.PixelData()[0]);
|
||||
for (int i = 1; i < kWidth * kHeight; ++i) {
|
||||
const uint8_t pixel_value = output_image.PixelData()[i];
|
||||
ASSERT_EQ(0, pixel_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -61,12 +61,12 @@ constexpr char kSessionBundleTag[] = "SESSION_BUNDLE";
|
||||
// overload GPU/TPU/...
|
||||
class SimpleSemaphore {
|
||||
public:
|
||||
explicit SimpleSemaphore(uint32 initial_count) : count_(initial_count) {}
|
||||
explicit SimpleSemaphore(uint32_t initial_count) : count_(initial_count) {}
|
||||
SimpleSemaphore(const SimpleSemaphore&) = delete;
|
||||
SimpleSemaphore(SimpleSemaphore&&) = delete;
|
||||
|
||||
// Acquires the semaphore by certain amount.
|
||||
void Acquire(uint32 amount) {
|
||||
void Acquire(uint32_t amount) {
|
||||
mutex_.Lock();
|
||||
while (count_ < amount) {
|
||||
cond_.Wait(&mutex_);
|
||||
@@ -76,7 +76,7 @@ class SimpleSemaphore {
|
||||
}
|
||||
|
||||
// Releases the semaphore by certain amount.
|
||||
void Release(uint32 amount) {
|
||||
void Release(uint32_t amount) {
|
||||
mutex_.Lock();
|
||||
count_ += amount;
|
||||
cond_.SignalAll();
|
||||
@@ -84,7 +84,7 @@ class SimpleSemaphore {
|
||||
}
|
||||
|
||||
private:
|
||||
uint32 count_;
|
||||
uint32_t count_;
|
||||
absl::Mutex mutex_;
|
||||
absl::CondVar cond_;
|
||||
};
|
||||
@@ -488,7 +488,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
// necessary.
|
||||
absl::Status OutputBatch(CalculatorContext* cc,
|
||||
std::unique_ptr<InferenceState> inference_state) {
|
||||
const int64 start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
const int64_t start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
std::vector<std::pair<mediapipe::ProtoString, tf::Tensor>> input_tensors;
|
||||
|
||||
for (auto& keyed_tensors : inference_state->input_tensor_batches_) {
|
||||
@@ -544,7 +544,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
get_session_run_throttle(options_.max_concurrent_session_runs());
|
||||
session_run_throttle->Acquire(1);
|
||||
}
|
||||
const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
const int64_t run_start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
tf::Status tf_status;
|
||||
{
|
||||
#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
|
||||
@@ -562,7 +562,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
// informative error message.
|
||||
RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
|
||||
|
||||
const int64 run_end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
const int64_t run_end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
cc->GetCounter(kTotalSessionRunsTimeUsecsCounterSuffix)
|
||||
->IncrementBy(run_end_time - run_start_time);
|
||||
cc->GetCounter(kTotalNumSessionRunsCounterSuffix)->Increment();
|
||||
@@ -611,7 +611,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
}
|
||||
|
||||
// Get end time and report.
|
||||
const int64 end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
const int64_t end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
cc->GetCounter(kTotalUsecsCounterSuffix)
|
||||
->IncrementBy(end_time - start_time);
|
||||
cc->GetCounter(kTotalProcessedTimestampsCounterSuffix)
|
||||
@@ -650,7 +650,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
|
||||
// The static singleton semaphore to throttle concurrent session runs.
|
||||
static SimpleSemaphore* get_session_run_throttle(
|
||||
int32 max_concurrent_session_runs) {
|
||||
int32_t max_concurrent_session_runs) {
|
||||
static SimpleSemaphore* session_run_throttle =
|
||||
new SimpleSemaphore(max_concurrent_session_runs);
|
||||
return session_run_throttle;
|
||||
|
||||
@@ -197,15 +197,15 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
|
||||
// timestamp and the associated feature. This information is used in process
|
||||
// to output batches of packets in order.
|
||||
timestamps_.clear();
|
||||
int64 last_timestamp_seen = Timestamp::PreStream().Value();
|
||||
int64_t last_timestamp_seen = Timestamp::PreStream().Value();
|
||||
first_timestamp_seen_ = Timestamp::OneOverPostStream().Value();
|
||||
for (const auto& map_kv : sequence_->feature_lists().feature_list()) {
|
||||
if (absl::StrContains(map_kv.first, "/timestamp")) {
|
||||
LOG(INFO) << "Found feature timestamps: " << map_kv.first
|
||||
<< " with size: " << map_kv.second.feature_size();
|
||||
int64 recent_timestamp = Timestamp::PreStream().Value();
|
||||
int64_t recent_timestamp = Timestamp::PreStream().Value();
|
||||
for (int i = 0; i < map_kv.second.feature_size(); ++i) {
|
||||
int64 next_timestamp =
|
||||
int64_t next_timestamp =
|
||||
mpms::GetInt64sAt(*sequence_, map_kv.first, i).Get(0);
|
||||
RET_CHECK_GT(next_timestamp, recent_timestamp)
|
||||
<< "Timestamps must be sequential. If you're seeing this message "
|
||||
@@ -361,8 +361,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
|
||||
// any particular call to Process(). At the every end, we output the
|
||||
// poststream packets. If we only have poststream packets,
|
||||
// last_timestamp_key_ will be empty.
|
||||
int64 start_timestamp = 0;
|
||||
int64 end_timestamp = 0;
|
||||
int64_t start_timestamp = 0;
|
||||
int64_t end_timestamp = 0;
|
||||
if (last_timestamp_key_.empty() || process_poststream_) {
|
||||
process_poststream_ = true;
|
||||
start_timestamp = Timestamp::PostStream().Value();
|
||||
@@ -481,14 +481,14 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
|
||||
// Store a map from the keys for each stream to the timestamps for each
|
||||
// key. This allows us to identify which packets to output for each stream
|
||||
// for timestamps within a given time window.
|
||||
std::map<std::string, std::vector<int64>> timestamps_;
|
||||
std::map<std::string, std::vector<int64_t>> timestamps_;
|
||||
// Store the stream with the latest timestamp in the SequenceExample.
|
||||
std::string last_timestamp_key_;
|
||||
// Store the index of the current timestamp. Will be less than
|
||||
// timestamps_[last_timestamp_key_].size().
|
||||
int current_timestamp_index_;
|
||||
// Store the very first timestamp, so we output everything on the first frame.
|
||||
int64 first_timestamp_seen_;
|
||||
int64_t first_timestamp_seen_;
|
||||
// List of keypoint names.
|
||||
std::vector<std::string> keypoint_names_;
|
||||
// Default keypoint location when missing.
|
||||
|
||||
@@ -54,7 +54,7 @@ class VectorToTensorFloatCalculatorTest : public ::testing::Test {
|
||||
}
|
||||
}
|
||||
|
||||
const int64 time = 1234;
|
||||
const int64_t time = 1234;
|
||||
runner_->MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(input.release()).At(Timestamp(time)));
|
||||
|
||||
@@ -91,7 +91,7 @@ TEST_F(VectorToTensorFloatCalculatorTest, ConvertsFromVectorFloat) {
|
||||
// 2^i can be represented exactly in floating point numbers if 'i' is small.
|
||||
input->at(i) = static_cast<float>(1 << i);
|
||||
}
|
||||
const int64 time = 1234;
|
||||
const int64_t time = 1234;
|
||||
runner_->MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(input.release()).At(Timestamp(time)));
|
||||
|
||||
|
||||
@@ -899,16 +899,77 @@ mediapipe_proto_library(
|
||||
cc_library(
|
||||
name = "landmarks_smoothing_calculator",
|
||||
srcs = ["landmarks_smoothing_calculator.cc"],
|
||||
hdrs = ["landmarks_smoothing_calculator.h"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_cc_proto",
|
||||
":landmarks_smoothing_calculator_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "landmarks_smoothing_calculator_utils",
|
||||
srcs = ["landmarks_smoothing_calculator_utils.cc"],
|
||||
hdrs = ["landmarks_smoothing_calculator_utils.h"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util/filtering:one_euro_filter",
|
||||
"//mediapipe/util/filtering:relative_velocity_filter",
|
||||
"@com_google_absl//absl/algorithm:container",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "landmarks_smoothing_calculator_utils_test",
|
||||
size = "small",
|
||||
srcs = ["landmarks_smoothing_calculator_utils_test.cc"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_utils",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "multi_landmarks_smoothing_calculator",
|
||||
srcs = ["multi_landmarks_smoothing_calculator.cc"],
|
||||
hdrs = ["multi_landmarks_smoothing_calculator.h"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_cc_proto",
|
||||
":landmarks_smoothing_calculator_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "multi_world_landmarks_smoothing_calculator",
|
||||
srcs = ["multi_world_landmarks_smoothing_calculator.cc"],
|
||||
hdrs = ["multi_world_landmarks_smoothing_calculator.h"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_cc_proto",
|
||||
":landmarks_smoothing_calculator_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -1285,12 +1346,14 @@ cc_library(
|
||||
srcs = ["flat_color_image_calculator.cc"],
|
||||
deps = [
|
||||
":flat_color_image_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util:color_cc_proto",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/strings",
|
||||
|
||||
@@ -172,7 +172,7 @@ class AnnotationOverlayCalculator : public CalculatorBase {
|
||||
REGISTER_CALCULATOR(AnnotationOverlayCalculator);
|
||||
|
||||
absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) {
|
||||
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||
|
||||
bool use_gpu = false;
|
||||
|
||||
@@ -189,13 +189,13 @@ absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (cc->Inputs().HasTag(kGpuBufferTag)) {
|
||||
cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
|
||||
CHECK(cc->Outputs().HasTag(kGpuBufferTag));
|
||||
RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag));
|
||||
use_gpu = true;
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
if (cc->Inputs().HasTag(kImageFrameTag)) {
|
||||
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
|
||||
CHECK(cc->Outputs().HasTag(kImageFrameTag));
|
||||
RET_CHECK(cc->Outputs().HasTag(kImageFrameTag));
|
||||
}
|
||||
|
||||
// Data streams to render.
|
||||
@@ -471,7 +471,7 @@ absl::Status AnnotationOverlayCalculator::CreateRenderTargetCpu(
|
||||
auto input_mat = formats::MatView(&input_frame);
|
||||
if (input_frame.Format() == ImageFormat::GRAY8) {
|
||||
cv::Mat rgb_mat;
|
||||
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
|
||||
cv::cvtColor(input_mat, rgb_mat, cv::COLOR_GRAY2RGB);
|
||||
rgb_mat.copyTo(*image_mat);
|
||||
} else {
|
||||
input_mat.copyTo(*image_mat);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -15,14 +15,13 @@
|
||||
#include <memory>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/util/flat_color_image_calculator.pb.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
#include "mediapipe/util/color.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
@@ -32,6 +31,7 @@ namespace {
|
||||
using ::mediapipe::api2::Input;
|
||||
using ::mediapipe::api2::Node;
|
||||
using ::mediapipe::api2::Output;
|
||||
using ::mediapipe::api2::SideOutput;
|
||||
} // namespace
|
||||
|
||||
// A calculator for generating an image filled with a single color.
|
||||
@@ -45,7 +45,8 @@ using ::mediapipe::api2::Output;
|
||||
//
|
||||
// Outputs:
|
||||
// IMAGE (Image)
|
||||
// Image filled with the requested color.
|
||||
// Image filled with the requested color. Can be either an output_stream
|
||||
// or an output_side_packet.
|
||||
//
|
||||
// Example useage:
|
||||
// node {
|
||||
@@ -68,9 +69,10 @@ class FlatColorImageCalculator : public Node {
|
||||
public:
|
||||
static constexpr Input<Image>::Optional kInImage{"IMAGE"};
|
||||
static constexpr Input<Color>::Optional kInColor{"COLOR"};
|
||||
static constexpr Output<Image> kOutImage{"IMAGE"};
|
||||
static constexpr Output<Image>::Optional kOutImage{"IMAGE"};
|
||||
static constexpr SideOutput<Image>::Optional kOutSideImage{"IMAGE"};
|
||||
|
||||
MEDIAPIPE_NODE_CONTRACT(kInImage, kInColor, kOutImage);
|
||||
MEDIAPIPE_NODE_CONTRACT(kInImage, kInColor, kOutImage, kOutSideImage);
|
||||
|
||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||
const auto& options = cc->Options<FlatColorImageCalculatorOptions>();
|
||||
@@ -81,6 +83,13 @@ class FlatColorImageCalculator : public Node {
|
||||
RET_CHECK(kInColor(cc).IsConnected() ^ options.has_color())
|
||||
<< "Either set COLOR input stream, or set through options";
|
||||
|
||||
RET_CHECK(kOutImage(cc).IsConnected() ^ kOutSideImage(cc).IsConnected())
|
||||
<< "Set IMAGE either as output stream, or as output side packet";
|
||||
|
||||
RET_CHECK(!kOutSideImage(cc).IsConnected() ||
|
||||
(options.has_output_height() && options.has_output_width()))
|
||||
<< "Set size through options, when setting IMAGE as output side packet";
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -88,6 +97,9 @@ class FlatColorImageCalculator : public Node {
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
std::optional<std::shared_ptr<ImageFrame>> CreateOutputFrame(
|
||||
CalculatorContext* cc);
|
||||
|
||||
bool use_dimension_from_option_ = false;
|
||||
bool use_color_from_option_ = false;
|
||||
};
|
||||
@@ -96,10 +108,31 @@ MEDIAPIPE_REGISTER_NODE(FlatColorImageCalculator);
|
||||
absl::Status FlatColorImageCalculator::Open(CalculatorContext* cc) {
|
||||
use_dimension_from_option_ = !kInImage(cc).IsConnected();
|
||||
use_color_from_option_ = !kInColor(cc).IsConnected();
|
||||
|
||||
if (!kOutImage(cc).IsConnected()) {
|
||||
std::optional<std::shared_ptr<ImageFrame>> output_frame =
|
||||
CreateOutputFrame(cc);
|
||||
if (output_frame.has_value()) {
|
||||
kOutSideImage(cc).Set(Image(output_frame.value()));
|
||||
}
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status FlatColorImageCalculator::Process(CalculatorContext* cc) {
|
||||
if (kOutImage(cc).IsConnected()) {
|
||||
std::optional<std::shared_ptr<ImageFrame>> output_frame =
|
||||
CreateOutputFrame(cc);
|
||||
if (output_frame.has_value()) {
|
||||
kOutImage(cc).Send(Image(output_frame.value()));
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
std::optional<std::shared_ptr<ImageFrame>>
|
||||
FlatColorImageCalculator::CreateOutputFrame(CalculatorContext* cc) {
|
||||
const auto& options = cc->Options<FlatColorImageCalculatorOptions>();
|
||||
|
||||
int output_height = -1;
|
||||
@@ -112,7 +145,7 @@ absl::Status FlatColorImageCalculator::Process(CalculatorContext* cc) {
|
||||
output_height = input_image.height();
|
||||
output_width = input_image.width();
|
||||
} else {
|
||||
return absl::OkStatus();
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
Color color;
|
||||
@@ -121,7 +154,7 @@ absl::Status FlatColorImageCalculator::Process(CalculatorContext* cc) {
|
||||
} else if (!kInColor(cc).IsEmpty()) {
|
||||
color = kInColor(cc).Get();
|
||||
} else {
|
||||
return absl::OkStatus();
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto output_frame = std::make_shared<ImageFrame>(ImageFormat::SRGB,
|
||||
@@ -130,9 +163,7 @@ absl::Status FlatColorImageCalculator::Process(CalculatorContext* cc) {
|
||||
|
||||
output_mat.setTo(cv::Scalar(color.r(), color.g(), color.b()));
|
||||
|
||||
kOutImage(cc).Send(Image(output_frame));
|
||||
|
||||
return absl::OkStatus();
|
||||
return output_frame;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -113,6 +113,35 @@ TEST(FlatColorImageCalculatorTest, SpecifyDimensionThroughOptions) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(FlatColorImageCalculatorTest, ProducesOutputSidePacket) {
|
||||
CalculatorRunner runner(R"pb(
|
||||
calculator: "FlatColorImageCalculator"
|
||||
output_side_packet: "IMAGE:out_packet"
|
||||
options {
|
||||
[mediapipe.FlatColorImageCalculatorOptions.ext] {
|
||||
output_width: 1
|
||||
output_height: 1
|
||||
color: {
|
||||
r: 100,
|
||||
g: 200,
|
||||
b: 255,
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const auto& image = runner.OutputSidePackets().Tag(kImageTag).Get<Image>();
|
||||
EXPECT_EQ(image.width(), 1);
|
||||
EXPECT_EQ(image.height(), 1);
|
||||
auto image_frame = image.GetImageFrameSharedPtr();
|
||||
const uint8_t* pixel_data = image_frame->PixelData();
|
||||
EXPECT_EQ(pixel_data[0], 100);
|
||||
EXPECT_EQ(pixel_data[1], 200);
|
||||
EXPECT_EQ(pixel_data[2], 255);
|
||||
}
|
||||
|
||||
TEST(FlatColorImageCalculatorTest, FailureMissingDimension) {
|
||||
CalculatorRunner runner(R"pb(
|
||||
calculator: "FlatColorImageCalculator"
|
||||
@@ -206,5 +235,56 @@ TEST(FlatColorImageCalculatorTest, FailureDuplicateColor) {
|
||||
HasSubstr("Either set COLOR input stream"));
|
||||
}
|
||||
|
||||
TEST(FlatColorImageCalculatorTest, FailureDuplicateOutputs) {
|
||||
CalculatorRunner runner(R"pb(
|
||||
calculator: "FlatColorImageCalculator"
|
||||
output_stream: "IMAGE:out_image"
|
||||
output_side_packet: "IMAGE:out_packet"
|
||||
options {
|
||||
[mediapipe.FlatColorImageCalculatorOptions.ext] {
|
||||
output_width: 1
|
||||
output_height: 1
|
||||
color: {
|
||||
r: 100,
|
||||
g: 200,
|
||||
b: 255,
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
|
||||
ASSERT_THAT(
|
||||
runner.Run().message(),
|
||||
HasSubstr("Set IMAGE either as output stream, or as output side packet"));
|
||||
}
|
||||
|
||||
TEST(FlatColorImageCalculatorTest, FailureSettingInputImageOnOutputSidePacket) {
|
||||
CalculatorRunner runner(R"pb(
|
||||
calculator: "FlatColorImageCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
output_side_packet: "IMAGE:out_packet"
|
||||
options {
|
||||
[mediapipe.FlatColorImageCalculatorOptions.ext] {
|
||||
color: {
|
||||
r: 100,
|
||||
g: 200,
|
||||
b: 255,
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
|
||||
auto image_frame = std::make_shared<ImageFrame>(ImageFormat::SRGB,
|
||||
kImageWidth, kImageHeight);
|
||||
|
||||
for (int ts = 0; ts < 3; ++ts) {
|
||||
runner.MutableInputs()->Tag(kImageTag).packets.push_back(
|
||||
MakePacket<Image>(image_frame).At(Timestamp(ts)));
|
||||
}
|
||||
ASSERT_THAT(runner.Run().message(),
|
||||
HasSubstr("Set size through options, when setting IMAGE as "
|
||||
"output side packet"));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -12,471 +12,105 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/util/filtering/one_euro_filter.h"
|
||||
#include "mediapipe/util/filtering/relative_velocity_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
|
||||
constexpr char kLandmarksTag[] = "LANDMARKS";
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
constexpr char kObjectScaleRoiTag[] = "OBJECT_SCALE_ROI";
|
||||
constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
|
||||
constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
|
||||
|
||||
using ::mediapipe::NormalizedRect;
|
||||
using mediapipe::OneEuroFilter;
|
||||
using ::mediapipe::Rect;
|
||||
using mediapipe::RelativeVelocityFilter;
|
||||
|
||||
void NormalizedLandmarksToLandmarks(
|
||||
const NormalizedLandmarkList& norm_landmarks, const int image_width,
|
||||
const int image_height, LandmarkList* landmarks) {
|
||||
for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
|
||||
const auto& norm_landmark = norm_landmarks.landmark(i);
|
||||
|
||||
auto* landmark = landmarks->add_landmark();
|
||||
landmark->set_x(norm_landmark.x() * image_width);
|
||||
landmark->set_y(norm_landmark.y() * image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
landmark->set_z(norm_landmark.z() * image_width);
|
||||
landmark->set_visibility(norm_landmark.visibility());
|
||||
landmark->set_presence(norm_landmark.presence());
|
||||
}
|
||||
}
|
||||
|
||||
void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
|
||||
const int image_width,
|
||||
const int image_height,
|
||||
NormalizedLandmarkList* norm_landmarks) {
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const auto& landmark = landmarks.landmark(i);
|
||||
|
||||
auto* norm_landmark = norm_landmarks->add_landmark();
|
||||
norm_landmark->set_x(landmark.x() / image_width);
|
||||
norm_landmark->set_y(landmark.y() / image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
norm_landmark->set_z(landmark.z() / image_width);
|
||||
norm_landmark->set_visibility(landmark.visibility());
|
||||
norm_landmark->set_presence(landmark.presence());
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate object scale to use its inverse value as velocity scale for
|
||||
// RelativeVelocityFilter. If value will be too small (less than
|
||||
// `options_.min_allowed_object_scale`) smoothing will be disabled and
|
||||
// landmarks will be returned as is.
|
||||
// Object scale is calculated as average between bounding box width and height
|
||||
// with sides parallel to axis.
|
||||
float GetObjectScale(const LandmarkList& landmarks) {
|
||||
const auto& lm_minmax_x = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.x() < b.x(); });
|
||||
const float x_min = lm_minmax_x.first->x();
|
||||
const float x_max = lm_minmax_x.second->x();
|
||||
|
||||
const auto& lm_minmax_y = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.y() < b.y(); });
|
||||
const float y_min = lm_minmax_y.first->y();
|
||||
const float y_max = lm_minmax_y.second->y();
|
||||
|
||||
const float object_width = x_max - x_min;
|
||||
const float object_height = y_max - y_min;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
|
||||
float GetObjectScale(const NormalizedRect& roi, const int image_width,
|
||||
const int image_height) {
|
||||
const float object_width = roi.width() * image_width;
|
||||
const float object_height = roi.height() * image_height;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
|
||||
float GetObjectScale(const Rect& roi) {
|
||||
return (roi.width() + roi.height()) / 2.0f;
|
||||
}
|
||||
|
||||
// Abstract class for various landmarks filters.
|
||||
class LandmarksFilter {
|
||||
public:
|
||||
virtual ~LandmarksFilter() = default;
|
||||
|
||||
virtual absl::Status Reset() { return absl::OkStatus(); }
|
||||
|
||||
virtual absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList* out_landmarks) = 0;
|
||||
};
|
||||
|
||||
// Returns landmarks as is without smoothing.
|
||||
class NoFilter : public LandmarksFilter {
|
||||
public:
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList* out_landmarks) override {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
// Please check RelativeVelocityFilter documentation for details.
|
||||
class VelocityFilter : public LandmarksFilter {
|
||||
public:
|
||||
VelocityFilter(int window_size, float velocity_scale,
|
||||
float min_allowed_object_scale, bool disable_value_scaling)
|
||||
: window_size_(window_size),
|
||||
velocity_scale_(velocity_scale),
|
||||
min_allowed_object_scale_(min_allowed_object_scale),
|
||||
disable_value_scaling_(disable_value_scaling) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList* out_landmarks) override {
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
float value_scale = 1.0f;
|
||||
if (!disable_value_scaling_) {
|
||||
const float object_scale =
|
||||
object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
value_scale = 1.0f / object_scale;
|
||||
}
|
||||
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks->add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(
|
||||
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
|
||||
out_landmark->set_y(
|
||||
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
|
||||
out_landmark->set_z(
|
||||
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
x_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
y_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
z_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
int window_size_;
|
||||
float velocity_scale_;
|
||||
float min_allowed_object_scale_;
|
||||
bool disable_value_scaling_;
|
||||
|
||||
std::vector<RelativeVelocityFilter> x_filters_;
|
||||
std::vector<RelativeVelocityFilter> y_filters_;
|
||||
std::vector<RelativeVelocityFilter> z_filters_;
|
||||
};
|
||||
|
||||
// Please check OneEuroFilter documentation for details.
|
||||
class OneEuroFilterImpl : public LandmarksFilter {
|
||||
public:
|
||||
OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
|
||||
double derivate_cutoff, float min_allowed_object_scale,
|
||||
bool disable_value_scaling)
|
||||
: frequency_(frequency),
|
||||
min_cutoff_(min_cutoff),
|
||||
beta_(beta),
|
||||
derivate_cutoff_(derivate_cutoff),
|
||||
min_allowed_object_scale_(min_allowed_object_scale),
|
||||
disable_value_scaling_(disable_value_scaling) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList* out_landmarks) override {
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
float value_scale = 1.0f;
|
||||
if (!disable_value_scaling_) {
|
||||
const float object_scale =
|
||||
object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
value_scale = 1.0f / object_scale;
|
||||
}
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks->add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(
|
||||
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
|
||||
out_landmark->set_y(
|
||||
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
|
||||
out_landmark->set_z(
|
||||
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_landmarks; ++i) {
|
||||
x_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
y_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
z_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
double frequency_;
|
||||
double min_cutoff_;
|
||||
double beta_;
|
||||
double derivate_cutoff_;
|
||||
double min_allowed_object_scale_;
|
||||
bool disable_value_scaling_;
|
||||
|
||||
std::vector<OneEuroFilter> x_filters_;
|
||||
std::vector<OneEuroFilter> y_filters_;
|
||||
std::vector<OneEuroFilter> z_filters_;
|
||||
};
|
||||
using ::mediapipe::landmarks_smoothing::GetObjectScale;
|
||||
using ::mediapipe::landmarks_smoothing::InitializeLandmarksFilter;
|
||||
using ::mediapipe::landmarks_smoothing::LandmarksFilter;
|
||||
using ::mediapipe::landmarks_smoothing::LandmarksToNormalizedLandmarks;
|
||||
using ::mediapipe::landmarks_smoothing::NormalizedLandmarksToLandmarks;
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
//
|
||||
// Inputs:
|
||||
// NORM_LANDMARKS: A NormalizedLandmarkList of landmarks you want to smooth.
|
||||
// IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
|
||||
// Required to perform all computations in absolute coordinates to avoid any
|
||||
// influence of normalized values.
|
||||
// OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
|
||||
// input landmarks) used to determine the object scale for some of the
|
||||
// filters. If not provided - object scale will be calculated from
|
||||
// landmarks.
|
||||
//
|
||||
// Outputs:
|
||||
// NORM_FILTERED_LANDMARKS: A NormalizedLandmarkList of smoothed landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "LandmarksSmoothingCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:pose_landmarks"
|
||||
// input_stream: "IMAGE_SIZE:image_size"
|
||||
// input_stream: "OBJECT_SCALE_ROI:roi"
|
||||
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
|
||||
// options: {
|
||||
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
// velocity_filter: {
|
||||
// window_size: 5
|
||||
// velocity_scale: 10.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class LandmarksSmoothingCalculator : public CalculatorBase {
|
||||
class LandmarksSmoothingCalculatorImpl
|
||||
: public NodeImpl<LandmarksSmoothingCalculator> {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
absl::Status Open(CalculatorContext* cc) override {
|
||||
ASSIGN_OR_RETURN(landmarks_filter_,
|
||||
InitializeLandmarksFilter(
|
||||
cc->Options<LandmarksSmoothingCalculatorOptions>()));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if ((kInNormLandmarks(cc).IsConnected() &&
|
||||
kInNormLandmarks(cc).IsEmpty()) ||
|
||||
(kInLandmarks(cc).IsConnected() && kInLandmarks(cc).IsEmpty())) {
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
if (kInNormLandmarks(cc).IsConnected()) {
|
||||
const auto& in_norm_landmarks = kInNormLandmarks(cc).Get();
|
||||
|
||||
int image_width;
|
||||
int image_height;
|
||||
std::tie(image_width, image_height) = kImageSize(cc).Get();
|
||||
|
||||
absl::optional<float> object_scale;
|
||||
if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
|
||||
auto& roi = kObjectScaleRoi(cc).Get<NormalizedRect>();
|
||||
object_scale = GetObjectScale(roi, image_width, image_height);
|
||||
}
|
||||
|
||||
auto in_landmarks = absl::make_unique<LandmarkList>();
|
||||
NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width,
|
||||
image_height, *in_landmarks.get());
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
|
||||
*in_landmarks, timestamp, object_scale, *out_landmarks));
|
||||
|
||||
auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
|
||||
LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
|
||||
*out_norm_landmarks.get());
|
||||
|
||||
kOutNormLandmarks(cc).Send(std::move(out_norm_landmarks));
|
||||
} else {
|
||||
const auto& in_landmarks = kInLandmarks(cc).Get();
|
||||
|
||||
absl::optional<float> object_scale;
|
||||
if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
|
||||
auto& roi = kObjectScaleRoi(cc).Get<Rect>();
|
||||
object_scale = GetObjectScale(roi);
|
||||
}
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
|
||||
in_landmarks, timestamp, object_scale, *out_landmarks));
|
||||
|
||||
kOutLandmarks(cc).Send(std::move(out_landmarks));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<LandmarksFilter> landmarks_filter_;
|
||||
};
|
||||
REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
|
||||
|
||||
absl::Status LandmarksSmoothingCalculator::GetContract(CalculatorContract* cc) {
|
||||
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Set<NormalizedLandmarkList>();
|
||||
|
||||
if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
|
||||
cc->Inputs().Tag(kObjectScaleRoiTag).Set<NormalizedRect>();
|
||||
}
|
||||
} else {
|
||||
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
|
||||
cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
|
||||
|
||||
if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
|
||||
cc->Inputs().Tag(kObjectScaleRoiTag).Set<Rect>();
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
// Pick landmarks filter.
|
||||
const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
|
||||
if (options.has_no_filter()) {
|
||||
landmarks_filter_ = absl::make_unique<NoFilter>();
|
||||
} else if (options.has_velocity_filter()) {
|
||||
landmarks_filter_ = absl::make_unique<VelocityFilter>(
|
||||
options.velocity_filter().window_size(),
|
||||
options.velocity_filter().velocity_scale(),
|
||||
options.velocity_filter().min_allowed_object_scale(),
|
||||
options.velocity_filter().disable_value_scaling());
|
||||
} else if (options.has_one_euro_filter()) {
|
||||
landmarks_filter_ = absl::make_unique<OneEuroFilterImpl>(
|
||||
options.one_euro_filter().frequency(),
|
||||
options.one_euro_filter().min_cutoff(),
|
||||
options.one_euro_filter().beta(),
|
||||
options.one_euro_filter().derivate_cutoff(),
|
||||
options.one_euro_filter().min_allowed_object_scale(),
|
||||
options.one_euro_filter().disable_value_scaling());
|
||||
} else {
|
||||
RET_CHECK_FAIL()
|
||||
<< "Landmarks filter is either not specified or not supported";
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
|
||||
(cc->Inputs().HasTag(kLandmarksTag) &&
|
||||
cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
|
||||
const auto& in_norm_landmarks =
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
|
||||
int image_width;
|
||||
int image_height;
|
||||
std::tie(image_width, image_height) =
|
||||
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
|
||||
absl::optional<float> object_scale;
|
||||
if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
|
||||
!cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
|
||||
auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<NormalizedRect>();
|
||||
object_scale = GetObjectScale(roi, image_width, image_height);
|
||||
}
|
||||
|
||||
auto in_landmarks = absl::make_unique<LandmarkList>();
|
||||
NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width, image_height,
|
||||
in_landmarks.get());
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
|
||||
*in_landmarks, timestamp, object_scale, out_landmarks.get()));
|
||||
|
||||
auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
|
||||
LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
|
||||
out_norm_landmarks.get());
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Add(out_norm_landmarks.release(), cc->InputTimestamp());
|
||||
} else {
|
||||
const auto& in_landmarks =
|
||||
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
|
||||
|
||||
absl::optional<float> object_scale;
|
||||
if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
|
||||
!cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
|
||||
auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<Rect>();
|
||||
object_scale = GetObjectScale(roi);
|
||||
}
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
|
||||
in_landmarks, timestamp, object_scale, out_landmarks.get()));
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kFilteredLandmarksTag)
|
||||
.Add(out_landmarks.release(), cc->InputTimestamp());
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
MEDIAPIPE_NODE_IMPLEMENTATION(LandmarksSmoothingCalculatorImpl);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
//
|
||||
// Inputs:
|
||||
// NORM_LANDMARKS (optional): A NormalizedLandmarkList of landmarks you want
|
||||
// to smooth.
|
||||
// LANDMARKS (optional): A LandmarkList of landmarks you want to smooth.
|
||||
// IMAGE_SIZE (optional): A std::pair<int, int> represention of image width
|
||||
// and height. Required to perform all computations in absolute coordinates
|
||||
// when smoothing NORM_LANDMARKS to avoid any influence of normalized
|
||||
// values.
|
||||
// OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
|
||||
// input landmarks) used to determine the object scale for some of the
|
||||
// filters. If not provided - object scale will be calculated from
|
||||
// landmarks.
|
||||
//
|
||||
// Outputs:
|
||||
// NORM_FILTERED_LANDMARKS (optional): A NormalizedLandmarkList of smoothed
|
||||
// landmarks.
|
||||
// FILTERED_LANDMARKS (optional): A LandmarkList of smoothed landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "LandmarksSmoothingCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:landmarks"
|
||||
// input_stream: "IMAGE_SIZE:image_size"
|
||||
// input_stream: "OBJECT_SCALE_ROI:roi"
|
||||
// output_stream: "NORM_FILTERED_LANDMARKS:landmarks_filtered"
|
||||
// options: {
|
||||
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
// velocity_filter: {
|
||||
// window_size: 5
|
||||
// velocity_scale: 10.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class LandmarksSmoothingCalculator : public NodeIntf {
|
||||
public:
|
||||
static constexpr Input<mediapipe::NormalizedLandmarkList>::Optional
|
||||
kInNormLandmarks{"NORM_LANDMARKS"};
|
||||
static constexpr Input<mediapipe::LandmarkList>::Optional kInLandmarks{
|
||||
"LANDMARKS"};
|
||||
static constexpr Input<std::pair<int, int>>::Optional kImageSize{
|
||||
"IMAGE_SIZE"};
|
||||
static constexpr Input<OneOf<NormalizedRect, Rect>>::Optional kObjectScaleRoi{
|
||||
"OBJECT_SCALE_ROI"};
|
||||
static constexpr Output<mediapipe::NormalizedLandmarkList>::Optional
|
||||
kOutNormLandmarks{"NORM_FILTERED_LANDMARKS"};
|
||||
static constexpr Output<mediapipe::LandmarkList>::Optional kOutLandmarks{
|
||||
"FILTERED_LANDMARKS"};
|
||||
MEDIAPIPE_NODE_INTERFACE(LandmarksSmoothingCalculator, kInNormLandmarks,
|
||||
kInLandmarks, kImageSize, kObjectScaleRoi,
|
||||
kOutNormLandmarks, kOutLandmarks);
|
||||
|
||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||
RET_CHECK(kInNormLandmarks(cc).IsConnected() ^
|
||||
kInLandmarks(cc).IsConnected())
|
||||
<< "One and only one of NORM_LANDMARKS and LANDMARKS input is allowed";
|
||||
|
||||
// TODO: Verify scale ROI is of the same type as landmarks
|
||||
// that are being smoothed.
|
||||
|
||||
if (kInNormLandmarks(cc).IsConnected()) {
|
||||
RET_CHECK(kImageSize(cc).IsConnected());
|
||||
RET_CHECK(kOutNormLandmarks(cc).IsConnected());
|
||||
RET_CHECK(!kOutLandmarks(cc).IsConnected());
|
||||
} else {
|
||||
RET_CHECK(!kImageSize(cc).IsConnected());
|
||||
RET_CHECK(kOutLandmarks(cc).IsConnected());
|
||||
RET_CHECK(!kOutNormLandmarks(cc).IsConnected());
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
@@ -0,0 +1,375 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/util/filtering/one_euro_filter.h"
|
||||
#include "mediapipe/util/filtering/relative_velocity_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace landmarks_smoothing {
|
||||
|
||||
namespace {
|
||||
|
||||
using ::mediapipe::NormalizedRect;
|
||||
using ::mediapipe::OneEuroFilter;
|
||||
using ::mediapipe::Rect;
|
||||
using ::mediapipe::RelativeVelocityFilter;
|
||||
|
||||
// Estimate object scale to use its inverse value as velocity scale for
|
||||
// RelativeVelocityFilter. If value will be too small (less than
|
||||
// `options_.min_allowed_object_scale`) smoothing will be disabled and
|
||||
// landmarks will be returned as is.
|
||||
// Object scale is calculated as average between bounding box width and height
|
||||
// with sides parallel to axis.
|
||||
float GetObjectScale(const LandmarkList& landmarks) {
|
||||
const auto& lm_minmax_x = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.x() < b.x(); });
|
||||
const float x_min = lm_minmax_x.first->x();
|
||||
const float x_max = lm_minmax_x.second->x();
|
||||
|
||||
const auto& lm_minmax_y = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.y() < b.y(); });
|
||||
const float y_min = lm_minmax_y.first->y();
|
||||
const float y_max = lm_minmax_y.second->y();
|
||||
|
||||
const float object_width = x_max - x_min;
|
||||
const float object_height = y_max - y_min;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
|
||||
// Returns landmarks as is without smoothing.
|
||||
class NoFilter : public LandmarksFilter {
|
||||
public:
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList& out_landmarks) override {
|
||||
out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
// Please check RelativeVelocityFilter documentation for details.
|
||||
class VelocityFilter : public LandmarksFilter {
|
||||
public:
|
||||
VelocityFilter(int window_size, float velocity_scale,
|
||||
float min_allowed_object_scale, bool disable_value_scaling)
|
||||
: window_size_(window_size),
|
||||
velocity_scale_(velocity_scale),
|
||||
min_allowed_object_scale_(min_allowed_object_scale),
|
||||
disable_value_scaling_(disable_value_scaling) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList& out_landmarks) override {
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
float value_scale = 1.0f;
|
||||
if (!disable_value_scaling_) {
|
||||
const float object_scale =
|
||||
object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
value_scale = 1.0f / object_scale;
|
||||
}
|
||||
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks.add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(
|
||||
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
|
||||
out_landmark->set_y(
|
||||
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
|
||||
out_landmark->set_z(
|
||||
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
x_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
y_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
z_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
int window_size_;
|
||||
float velocity_scale_;
|
||||
float min_allowed_object_scale_;
|
||||
bool disable_value_scaling_;
|
||||
|
||||
std::vector<RelativeVelocityFilter> x_filters_;
|
||||
std::vector<RelativeVelocityFilter> y_filters_;
|
||||
std::vector<RelativeVelocityFilter> z_filters_;
|
||||
};
|
||||
|
||||
// Please check OneEuroFilter documentation for details.
|
||||
class OneEuroFilterImpl : public LandmarksFilter {
|
||||
public:
|
||||
OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
|
||||
double derivate_cutoff, float min_allowed_object_scale,
|
||||
bool disable_value_scaling)
|
||||
: frequency_(frequency),
|
||||
min_cutoff_(min_cutoff),
|
||||
beta_(beta),
|
||||
derivate_cutoff_(derivate_cutoff),
|
||||
min_allowed_object_scale_(min_allowed_object_scale),
|
||||
disable_value_scaling_(disable_value_scaling) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList& out_landmarks) override {
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
float value_scale = 1.0f;
|
||||
if (!disable_value_scaling_) {
|
||||
const float object_scale =
|
||||
object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
value_scale = 1.0f / object_scale;
|
||||
}
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks.add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(
|
||||
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
|
||||
out_landmark->set_y(
|
||||
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
|
||||
out_landmark->set_z(
|
||||
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_landmarks; ++i) {
|
||||
x_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
y_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
z_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
double frequency_;
|
||||
double min_cutoff_;
|
||||
double beta_;
|
||||
double derivate_cutoff_;
|
||||
double min_allowed_object_scale_;
|
||||
bool disable_value_scaling_;
|
||||
|
||||
std::vector<OneEuroFilter> x_filters_;
|
||||
std::vector<OneEuroFilter> y_filters_;
|
||||
std::vector<OneEuroFilter> z_filters_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void NormalizedLandmarksToLandmarks(
|
||||
const NormalizedLandmarkList& norm_landmarks, const int image_width,
|
||||
const int image_height, LandmarkList& landmarks) {
|
||||
for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
|
||||
const auto& norm_landmark = norm_landmarks.landmark(i);
|
||||
|
||||
auto* landmark = landmarks.add_landmark();
|
||||
landmark->set_x(norm_landmark.x() * image_width);
|
||||
landmark->set_y(norm_landmark.y() * image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
landmark->set_z(norm_landmark.z() * image_width);
|
||||
|
||||
if (norm_landmark.has_visibility()) {
|
||||
landmark->set_visibility(norm_landmark.visibility());
|
||||
} else {
|
||||
landmark->clear_visibility();
|
||||
}
|
||||
|
||||
if (norm_landmark.has_presence()) {
|
||||
landmark->set_presence(norm_landmark.presence());
|
||||
} else {
|
||||
landmark->clear_presence();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
|
||||
const int image_width,
|
||||
const int image_height,
|
||||
NormalizedLandmarkList& norm_landmarks) {
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const auto& landmark = landmarks.landmark(i);
|
||||
|
||||
auto* norm_landmark = norm_landmarks.add_landmark();
|
||||
norm_landmark->set_x(landmark.x() / image_width);
|
||||
norm_landmark->set_y(landmark.y() / image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
norm_landmark->set_z(landmark.z() / image_width);
|
||||
|
||||
if (landmark.has_visibility()) {
|
||||
norm_landmark->set_visibility(landmark.visibility());
|
||||
} else {
|
||||
norm_landmark->clear_visibility();
|
||||
}
|
||||
|
||||
if (landmark.has_presence()) {
|
||||
norm_landmark->set_presence(landmark.presence());
|
||||
} else {
|
||||
norm_landmark->clear_presence();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
float GetObjectScale(const NormalizedRect& roi, const int image_width,
|
||||
const int image_height) {
|
||||
const float object_width = roi.width() * image_width;
|
||||
const float object_height = roi.height() * image_height;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
|
||||
float GetObjectScale(const Rect& roi) {
|
||||
return (roi.width() + roi.height()) / 2.0f;
|
||||
}
|
||||
|
||||
absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
|
||||
const LandmarksSmoothingCalculatorOptions& options) {
|
||||
if (options.has_no_filter()) {
|
||||
return absl::make_unique<NoFilter>();
|
||||
} else if (options.has_velocity_filter()) {
|
||||
return absl::make_unique<VelocityFilter>(
|
||||
options.velocity_filter().window_size(),
|
||||
options.velocity_filter().velocity_scale(),
|
||||
options.velocity_filter().min_allowed_object_scale(),
|
||||
options.velocity_filter().disable_value_scaling());
|
||||
} else if (options.has_one_euro_filter()) {
|
||||
return absl::make_unique<OneEuroFilterImpl>(
|
||||
options.one_euro_filter().frequency(),
|
||||
options.one_euro_filter().min_cutoff(),
|
||||
options.one_euro_filter().beta(),
|
||||
options.one_euro_filter().derivate_cutoff(),
|
||||
options.one_euro_filter().min_allowed_object_scale(),
|
||||
options.one_euro_filter().disable_value_scaling());
|
||||
} else {
|
||||
RET_CHECK_FAIL()
|
||||
<< "Landmarks filter is either not specified or not supported";
|
||||
}
|
||||
}
|
||||
|
||||
absl::StatusOr<LandmarksFilter*> MultiLandmarkFilters::GetOrCreate(
|
||||
const int64_t tracking_id,
|
||||
const mediapipe::LandmarksSmoothingCalculatorOptions& options) {
|
||||
const auto it = filters_.find(tracking_id);
|
||||
if (it != filters_.end()) {
|
||||
return it->second.get();
|
||||
}
|
||||
|
||||
ASSIGN_OR_RETURN(auto landmarks_filter, InitializeLandmarksFilter(options));
|
||||
filters_[tracking_id] = std::move(landmarks_filter);
|
||||
return filters_[tracking_id].get();
|
||||
}
|
||||
|
||||
void MultiLandmarkFilters::ClearUnused(
|
||||
const std::vector<int64_t>& tracking_ids) {
|
||||
std::vector<int64_t> unused_tracking_ids;
|
||||
for (const auto& it : filters_) {
|
||||
bool unused = true;
|
||||
for (int64_t tracking_id : tracking_ids) {
|
||||
if (tracking_id == it.first) unused = false;
|
||||
}
|
||||
if (unused) unused_tracking_ids.push_back(it.first);
|
||||
}
|
||||
|
||||
for (int64_t tracking_id : unused_tracking_ids) {
|
||||
filters_.erase(tracking_id);
|
||||
}
|
||||
}
|
||||
|
||||
void MultiLandmarkFilters::Clear() { filters_.clear(); }
|
||||
|
||||
} // namespace landmarks_smoothing
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
|
||||
#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/util/filtering/one_euro_filter.h"
|
||||
#include "mediapipe/util/filtering/relative_velocity_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace landmarks_smoothing {
|
||||
|
||||
void NormalizedLandmarksToLandmarks(
|
||||
const mediapipe::NormalizedLandmarkList& norm_landmarks,
|
||||
const int image_width, const int image_height,
|
||||
mediapipe::LandmarkList& landmarks);
|
||||
|
||||
void LandmarksToNormalizedLandmarks(
|
||||
const mediapipe::LandmarkList& landmarks, const int image_width,
|
||||
const int image_height, mediapipe::NormalizedLandmarkList& norm_landmarks);
|
||||
|
||||
float GetObjectScale(const NormalizedRect& roi, const int image_width,
|
||||
const int image_height);
|
||||
|
||||
float GetObjectScale(const Rect& roi);
|
||||
|
||||
// Abstract class for various landmarks filters.
|
||||
class LandmarksFilter {
|
||||
public:
|
||||
virtual ~LandmarksFilter() = default;
|
||||
|
||||
virtual absl::Status Reset() { return absl::OkStatus(); }
|
||||
|
||||
virtual absl::Status Apply(const mediapipe::LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
mediapipe::LandmarkList& out_landmarks) = 0;
|
||||
};
|
||||
|
||||
absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
|
||||
const mediapipe::LandmarksSmoothingCalculatorOptions& options);
|
||||
|
||||
class MultiLandmarkFilters {
|
||||
public:
|
||||
virtual ~MultiLandmarkFilters() = default;
|
||||
|
||||
virtual absl::StatusOr<LandmarksFilter*> GetOrCreate(
|
||||
const int64_t tracking_id,
|
||||
const mediapipe::LandmarksSmoothingCalculatorOptions& options);
|
||||
|
||||
virtual void ClearUnused(const std::vector<int64_t>& tracking_ids);
|
||||
|
||||
virtual void Clear();
|
||||
|
||||
private:
|
||||
std::map<int64_t, std::unique_ptr<LandmarksFilter>> filters_;
|
||||
};
|
||||
|
||||
} // namespace landmarks_smoothing
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
|
||||
@@ -0,0 +1,118 @@
|
||||
/* Copyright 2023 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace landmarks_smoothing {
|
||||
namespace {
|
||||
|
||||
TEST(LandmarksSmoothingCalculatorUtilsTest, NormalizedLandmarksToLandmarks) {
|
||||
NormalizedLandmarkList norm_landmarks;
|
||||
NormalizedLandmark* norm_landmark = norm_landmarks.add_landmark();
|
||||
norm_landmark->set_x(0.1);
|
||||
norm_landmark->set_y(0.2);
|
||||
norm_landmark->set_z(0.3);
|
||||
norm_landmark->set_visibility(0.4);
|
||||
norm_landmark->set_presence(0.5);
|
||||
|
||||
LandmarkList landmarks;
|
||||
NormalizedLandmarksToLandmarks(norm_landmarks, /*image_width=*/10,
|
||||
/*image_height=*/10, landmarks);
|
||||
|
||||
EXPECT_EQ(landmarks.landmark_size(), 1);
|
||||
Landmark landmark = landmarks.landmark(0);
|
||||
EXPECT_NEAR(landmark.x(), 1.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.y(), 2.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.z(), 3.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.visibility(), 0.4, 1e-6);
|
||||
EXPECT_NEAR(landmark.presence(), 0.5, 1e-6);
|
||||
}
|
||||
|
||||
TEST(LandmarksSmoothingCalculatorUtilsTest,
|
||||
NormalizedLandmarksToLandmarks_EmptyVisibilityAndPresence) {
|
||||
NormalizedLandmarkList norm_landmarks;
|
||||
NormalizedLandmark* norm_landmark = norm_landmarks.add_landmark();
|
||||
norm_landmark->set_x(0.1);
|
||||
norm_landmark->set_y(0.2);
|
||||
norm_landmark->set_z(0.3);
|
||||
norm_landmark->clear_visibility();
|
||||
norm_landmark->clear_presence();
|
||||
|
||||
LandmarkList landmarks;
|
||||
NormalizedLandmarksToLandmarks(norm_landmarks, /*image_width=*/10,
|
||||
/*image_height=*/10, landmarks);
|
||||
|
||||
EXPECT_EQ(landmarks.landmark_size(), 1);
|
||||
Landmark landmark = landmarks.landmark(0);
|
||||
EXPECT_NEAR(landmark.x(), 1.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.y(), 2.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.z(), 3.0, 1e-6);
|
||||
EXPECT_FALSE(landmark.has_visibility());
|
||||
EXPECT_FALSE(landmark.has_presence());
|
||||
}
|
||||
|
||||
TEST(LandmarksSmoothingCalculatorUtilsTest, LandmarksToNormalizedLandmarks) {
|
||||
LandmarkList landmarks;
|
||||
Landmark* landmark = landmarks.add_landmark();
|
||||
landmark->set_x(1.0);
|
||||
landmark->set_y(2.0);
|
||||
landmark->set_z(3.0);
|
||||
landmark->set_visibility(0.4);
|
||||
landmark->set_presence(0.5);
|
||||
|
||||
NormalizedLandmarkList norm_landmarks;
|
||||
LandmarksToNormalizedLandmarks(landmarks, /*image_width=*/10,
|
||||
/*image_height=*/10, norm_landmarks);
|
||||
|
||||
EXPECT_EQ(norm_landmarks.landmark_size(), 1);
|
||||
NormalizedLandmark norm_landmark = norm_landmarks.landmark(0);
|
||||
EXPECT_NEAR(norm_landmark.x(), 0.1, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.y(), 0.2, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.z(), 0.3, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.visibility(), 0.4, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.presence(), 0.5, 1e-6);
|
||||
}
|
||||
|
||||
TEST(LandmarksSmoothingCalculatorUtilsTest,
|
||||
LandmarksToNormalizedLandmarks_EmptyVisibilityAndPresence) {
|
||||
LandmarkList landmarks;
|
||||
Landmark* landmark = landmarks.add_landmark();
|
||||
landmark->set_x(1.0);
|
||||
landmark->set_y(2.0);
|
||||
landmark->set_z(3.0);
|
||||
landmark->clear_visibility();
|
||||
landmark->clear_presence();
|
||||
|
||||
NormalizedLandmarkList norm_landmarks;
|
||||
LandmarksToNormalizedLandmarks(landmarks, /*image_width=*/10,
|
||||
/*image_height=*/10, norm_landmarks);
|
||||
|
||||
EXPECT_EQ(norm_landmarks.landmark_size(), 1);
|
||||
NormalizedLandmark norm_landmark = norm_landmarks.landmark(0);
|
||||
EXPECT_NEAR(norm_landmark.x(), 0.1, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.y(), 0.2, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.z(), 0.3, 1e-6);
|
||||
EXPECT_FALSE(norm_landmark.has_visibility());
|
||||
EXPECT_FALSE(norm_landmark.has_presence());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace landmarks_smoothing
|
||||
} // namespace mediapipe
|
||||
@@ -322,27 +322,30 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
options_.presence_threshold(), options_.connection_color(), thickness,
|
||||
/*normalized=*/false, render_data.get());
|
||||
}
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const Landmark& landmark = landmarks.landmark(i);
|
||||
if (options_.render_landmarks()) {
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const Landmark& landmark = landmarks.landmark(i);
|
||||
|
||||
if (!IsLandmarkVisibleAndPresent<Landmark>(
|
||||
landmark, options_.utilize_visibility(),
|
||||
options_.visibility_threshold(), options_.utilize_presence(),
|
||||
options_.presence_threshold())) {
|
||||
continue;
|
||||
}
|
||||
if (!IsLandmarkVisibleAndPresent<Landmark>(
|
||||
landmark, options_.utilize_visibility(),
|
||||
options_.visibility_threshold(), options_.utilize_presence(),
|
||||
options_.presence_threshold())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto* landmark_data_render = AddPointRenderData(
|
||||
options_.landmark_color(), thickness, render_data.get());
|
||||
if (visualize_depth) {
|
||||
SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
|
||||
options_.min_depth_circle_thickness(),
|
||||
options_.max_depth_circle_thickness());
|
||||
auto* landmark_data_render = AddPointRenderData(
|
||||
options_.landmark_color(), thickness, render_data.get());
|
||||
if (visualize_depth) {
|
||||
SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
|
||||
landmark_data_render,
|
||||
options_.min_depth_circle_thickness(),
|
||||
options_.max_depth_circle_thickness());
|
||||
}
|
||||
auto* landmark_data = landmark_data_render->mutable_point();
|
||||
landmark_data->set_normalized(false);
|
||||
landmark_data->set_x(landmark.x());
|
||||
landmark_data->set_y(landmark.y());
|
||||
}
|
||||
auto* landmark_data = landmark_data_render->mutable_point();
|
||||
landmark_data->set_normalized(false);
|
||||
landmark_data->set_x(landmark.x());
|
||||
landmark_data->set_y(landmark.y());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -368,27 +371,30 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
|
||||
options_.presence_threshold(), options_.connection_color(), thickness,
|
||||
/*normalized=*/true, render_data.get());
|
||||
}
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const NormalizedLandmark& landmark = landmarks.landmark(i);
|
||||
if (options_.render_landmarks()) {
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const NormalizedLandmark& landmark = landmarks.landmark(i);
|
||||
|
||||
if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
|
||||
landmark, options_.utilize_visibility(),
|
||||
options_.visibility_threshold(), options_.utilize_presence(),
|
||||
options_.presence_threshold())) {
|
||||
continue;
|
||||
}
|
||||
if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
|
||||
landmark, options_.utilize_visibility(),
|
||||
options_.visibility_threshold(), options_.utilize_presence(),
|
||||
options_.presence_threshold())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto* landmark_data_render = AddPointRenderData(
|
||||
options_.landmark_color(), thickness, render_data.get());
|
||||
if (visualize_depth) {
|
||||
SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
|
||||
options_.min_depth_circle_thickness(),
|
||||
options_.max_depth_circle_thickness());
|
||||
auto* landmark_data_render = AddPointRenderData(
|
||||
options_.landmark_color(), thickness, render_data.get());
|
||||
if (visualize_depth) {
|
||||
SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
|
||||
landmark_data_render,
|
||||
options_.min_depth_circle_thickness(),
|
||||
options_.max_depth_circle_thickness());
|
||||
}
|
||||
auto* landmark_data = landmark_data_render->mutable_point();
|
||||
landmark_data->set_normalized(true);
|
||||
landmark_data->set_x(landmark.x());
|
||||
landmark_data->set_y(landmark.y());
|
||||
}
|
||||
auto* landmark_data = landmark_data_render->mutable_point();
|
||||
landmark_data->set_normalized(true);
|
||||
landmark_data->set_x(landmark.x());
|
||||
landmark_data->set_y(landmark.y());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,10 @@ message LandmarksToRenderDataCalculatorOptions {
|
||||
|
||||
// Color of the landmarks.
|
||||
optional Color landmark_color = 2;
|
||||
|
||||
// Whether to render landmarks as points.
|
||||
optional bool render_landmarks = 14 [default = true];
|
||||
|
||||
// Color of the connections.
|
||||
optional Color connection_color = 3;
|
||||
|
||||
|
||||
@@ -18,6 +18,9 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option java_package = "com.google.mediapipe.calculator.proto";
|
||||
option java_outer_classname = "LogicCalculatorOptionsProto";
|
||||
|
||||
message LogicCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional LogicCalculatorOptions ext = 338731246;
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/util/multi_landmarks_smoothing_calculator.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
namespace {
|
||||
|
||||
using ::mediapipe::NormalizedRect;
|
||||
using ::mediapipe::landmarks_smoothing::GetObjectScale;
|
||||
using ::mediapipe::landmarks_smoothing::LandmarksToNormalizedLandmarks;
|
||||
using ::mediapipe::landmarks_smoothing::MultiLandmarkFilters;
|
||||
using ::mediapipe::landmarks_smoothing::NormalizedLandmarksToLandmarks;
|
||||
|
||||
} // namespace
|
||||
|
||||
class MultiLandmarksSmoothingCalculatorImpl
|
||||
: public NodeImpl<MultiLandmarksSmoothingCalculator> {
|
||||
public:
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if (kInNormLandmarks(cc).IsEmpty()) {
|
||||
multi_filters_.Clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
const auto& tracking_ids = kTrackingIds(cc).Get();
|
||||
multi_filters_.ClearUnused(tracking_ids);
|
||||
|
||||
const auto& in_norm_landmarks_vec = kInNormLandmarks(cc).Get();
|
||||
RET_CHECK_EQ(in_norm_landmarks_vec.size(), tracking_ids.size());
|
||||
|
||||
int image_width;
|
||||
int image_height;
|
||||
std::tie(image_width, image_height) = kImageSize(cc).Get();
|
||||
|
||||
std::optional<std::vector<NormalizedRect>> object_scale_roi_vec;
|
||||
if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
|
||||
object_scale_roi_vec = kObjectScaleRoi(cc).Get();
|
||||
RET_CHECK_EQ(object_scale_roi_vec.value().size(), tracking_ids.size());
|
||||
}
|
||||
|
||||
std::vector<NormalizedLandmarkList> out_norm_landmarks_vec;
|
||||
for (int i = 0; i < tracking_ids.size(); ++i) {
|
||||
LandmarkList in_landmarks;
|
||||
NormalizedLandmarksToLandmarks(in_norm_landmarks_vec[i], image_width,
|
||||
image_height, in_landmarks);
|
||||
|
||||
std::optional<float> object_scale;
|
||||
if (object_scale_roi_vec) {
|
||||
object_scale = GetObjectScale(object_scale_roi_vec.value()[i],
|
||||
image_width, image_height);
|
||||
}
|
||||
|
||||
ASSIGN_OR_RETURN(auto* landmarks_filter,
|
||||
multi_filters_.GetOrCreate(
|
||||
tracking_ids[i],
|
||||
cc->Options<LandmarksSmoothingCalculatorOptions>()));
|
||||
|
||||
LandmarkList out_landmarks;
|
||||
MP_RETURN_IF_ERROR(landmarks_filter->Apply(in_landmarks, timestamp,
|
||||
object_scale, out_landmarks));
|
||||
|
||||
NormalizedLandmarkList out_norm_landmarks;
|
||||
LandmarksToNormalizedLandmarks(out_landmarks, image_width, image_height,
|
||||
out_norm_landmarks);
|
||||
|
||||
out_norm_landmarks_vec.push_back(std::move(out_norm_landmarks));
|
||||
}
|
||||
|
||||
kOutNormLandmarks(cc).Send(std::move(out_norm_landmarks_vec));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
MultiLandmarkFilters multi_filters_;
|
||||
};
|
||||
MEDIAPIPE_NODE_IMPLEMENTATION(MultiLandmarksSmoothingCalculatorImpl);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
//
|
||||
// Inputs:
|
||||
// NORM_LANDMARKS: A std::vector<NormalizedLandmarkList> of landmarks you want
|
||||
// to smooth.
|
||||
// TRACKING_IDS: A std<int64_t> vector of tracking IDs used to associate
|
||||
// landmarks over time. When new ID arrives - calculator will initialize new
|
||||
// filter. When tracking ID is no longer provided - calculator will forget
|
||||
// smoothing state.
|
||||
// IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
|
||||
// Required to perform all computations in absolute coordinates to avoid any
|
||||
// influence of normalized values.
|
||||
// OBJECT_SCALE_ROI (optional): A std::vector<NormRect> used to determine the
|
||||
// object scale for some of the filters. If not provided - object scale will
|
||||
// be calculated from landmarks.
|
||||
//
|
||||
// Outputs:
|
||||
// NORM_FILTERED_LANDMARKS: A std::vector<NormalizedLandmarkList> of smoothed
|
||||
// landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "MultiLandmarksSmoothingCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:pose_landmarks"
|
||||
// input_stream: "IMAGE_SIZE:image_size"
|
||||
// input_stream: "OBJECT_SCALE_ROI:roi"
|
||||
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
|
||||
// options: {
|
||||
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
// velocity_filter: {
|
||||
// window_size: 5
|
||||
// velocity_scale: 10.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class MultiLandmarksSmoothingCalculator : public NodeIntf {
|
||||
public:
|
||||
static constexpr Input<std::vector<mediapipe::NormalizedLandmarkList>>
|
||||
kInNormLandmarks{"NORM_LANDMARKS"};
|
||||
static constexpr Input<std::vector<int64_t>> kTrackingIds{"TRACKING_IDS"};
|
||||
static constexpr Input<std::pair<int, int>> kImageSize{"IMAGE_SIZE"};
|
||||
static constexpr Input<std::vector<NormalizedRect>>::Optional kObjectScaleRoi{
|
||||
"OBJECT_SCALE_ROI"};
|
||||
static constexpr Output<std::vector<mediapipe::NormalizedLandmarkList>>
|
||||
kOutNormLandmarks{"NORM_FILTERED_LANDMARKS"};
|
||||
|
||||
MEDIAPIPE_NODE_INTERFACE(MultiLandmarksSmoothingCalculator, kInNormLandmarks,
|
||||
kTrackingIds, kImageSize, kObjectScaleRoi,
|
||||
kOutNormLandmarks);
|
||||
};
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/util/multi_world_landmarks_smoothing_calculator.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
namespace {
|
||||
|
||||
using ::mediapipe::Rect;
|
||||
using ::mediapipe::landmarks_smoothing::GetObjectScale;
|
||||
using ::mediapipe::landmarks_smoothing::MultiLandmarkFilters;
|
||||
|
||||
} // namespace
|
||||
|
||||
class MultiWorldLandmarksSmoothingCalculatorImpl
|
||||
: public NodeImpl<MultiWorldLandmarksSmoothingCalculator> {
|
||||
public:
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if (kInLandmarks(cc).IsEmpty()) {
|
||||
multi_filters_.Clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
const auto& tracking_ids = kTrackingIds(cc).Get();
|
||||
multi_filters_.ClearUnused(tracking_ids);
|
||||
|
||||
const auto& in_landmarks_vec = kInLandmarks(cc).Get();
|
||||
RET_CHECK_EQ(in_landmarks_vec.size(), tracking_ids.size());
|
||||
|
||||
std::optional<std::vector<Rect>> object_scale_roi_vec;
|
||||
if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
|
||||
object_scale_roi_vec = kObjectScaleRoi(cc).Get();
|
||||
RET_CHECK_EQ(object_scale_roi_vec.value().size(), tracking_ids.size());
|
||||
}
|
||||
|
||||
std::vector<LandmarkList> out_landmarks_vec;
|
||||
for (int i = 0; i < tracking_ids.size(); ++i) {
|
||||
const auto& in_landmarks = in_landmarks_vec[i];
|
||||
|
||||
std::optional<float> object_scale;
|
||||
if (object_scale_roi_vec) {
|
||||
object_scale = GetObjectScale(object_scale_roi_vec.value()[i]);
|
||||
}
|
||||
|
||||
ASSIGN_OR_RETURN(auto* landmarks_filter,
|
||||
multi_filters_.GetOrCreate(
|
||||
tracking_ids[i],
|
||||
cc->Options<LandmarksSmoothingCalculatorOptions>()));
|
||||
|
||||
LandmarkList out_landmarks;
|
||||
MP_RETURN_IF_ERROR(landmarks_filter->Apply(in_landmarks, timestamp,
|
||||
object_scale, out_landmarks));
|
||||
|
||||
out_landmarks_vec.push_back(std::move(out_landmarks));
|
||||
}
|
||||
|
||||
kOutLandmarks(cc).Send(std::move(out_landmarks_vec));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
MultiLandmarkFilters multi_filters_;
|
||||
};
|
||||
MEDIAPIPE_NODE_IMPLEMENTATION(MultiWorldLandmarksSmoothingCalculatorImpl);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
//
|
||||
// Inputs:
|
||||
// LANDMARKS: A std::vector<LandmarkList> of landmarks you want to
|
||||
// smooth.
|
||||
// TRACKING_IDS: A std<int64_t> vector of tracking IDs used to associate
|
||||
// landmarks over time. When new ID arrives - calculator will initialize new
|
||||
// filter. When tracking ID is no longer provided - calculator will forget
|
||||
// smoothing state.
|
||||
// OBJECT_SCALE_ROI (optional): A std::vector<Rect> used to determine the
|
||||
// object scale for some of the filters. If not provided - object scale will
|
||||
// be calculated from landmarks.
|
||||
//
|
||||
// Outputs:
|
||||
// FILTERED_LANDMARKS: A std::vector<LandmarkList> of smoothed landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "MultiWorldLandmarksSmoothingCalculator"
|
||||
// input_stream: "LANDMARKS:landmarks"
|
||||
// input_stream: "OBJECT_SCALE_ROI:roi"
|
||||
// output_stream: "FILTERED_LANDMARKS:landmarks_filtered"
|
||||
// options: {
|
||||
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
// velocity_filter: {
|
||||
// window_size: 5
|
||||
// velocity_scale: 10.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class MultiWorldLandmarksSmoothingCalculator : public NodeIntf {
|
||||
public:
|
||||
static constexpr Input<std::vector<mediapipe::LandmarkList>> kInLandmarks{
|
||||
"LANDMARKS"};
|
||||
static constexpr Input<std::vector<int64_t>> kTrackingIds{"TRACKING_IDS"};
|
||||
static constexpr Input<std::vector<Rect>>::Optional kObjectScaleRoi{
|
||||
"OBJECT_SCALE_ROI"};
|
||||
static constexpr Output<std::vector<mediapipe::LandmarkList>> kOutLandmarks{
|
||||
"FILTERED_LANDMARKS"};
|
||||
|
||||
MEDIAPIPE_NODE_INTERFACE(MultiWorldLandmarksSmoothingCalculator, kInLandmarks,
|
||||
kTrackingIds, kObjectScaleRoi, kOutLandmarks);
|
||||
};
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
@@ -124,7 +124,7 @@ absl::StatusOr<mediapipe::NormalizedLandmarkList> RefineLandmarksFromHeatMap(
|
||||
int center_row = out_lms.landmark(lm_index).y() * hm_height;
|
||||
// Point is outside of the image let's keep it intact.
|
||||
if (center_col < 0 || center_col >= hm_width || center_row < 0 ||
|
||||
center_col >= hm_height) {
|
||||
center_row >= hm_height) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -130,7 +130,6 @@ cc_library(
|
||||
"//mediapipe/framework/formats:video_stream_header",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:opencv_video",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:status_util",
|
||||
],
|
||||
@@ -341,7 +340,6 @@ cc_test(
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:test_util",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -367,7 +365,6 @@ cc_test(
|
||||
"//mediapipe/framework/port:opencv_video",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:test_util",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -451,7 +448,6 @@ cc_test(
|
||||
"//mediapipe/framework/tool:test_util",
|
||||
"//mediapipe/util/tracking:box_tracker_cc_proto",
|
||||
"//mediapipe/util/tracking:tracking_cc_proto",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
Binary file not shown.
+1
-1
@@ -1,6 +1,6 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip
|
||||
networkTimeout=10000
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user