Compare commits
@@ -87,6 +87,9 @@ build:ios_fat --config=ios
|
||||
build:ios_fat --ios_multi_cpus=armv7,arm64
|
||||
build:ios_fat --watchos_cpus=armv7k
|
||||
|
||||
build:ios_sim_fat --config=ios
|
||||
build:ios_sim_fat --ios_multi_cpus=x86_64,sim_arm64
|
||||
|
||||
build:darwin_x86_64 --apple_platform_type=macos
|
||||
build:darwin_x86_64 --macos_minimum_os=10.12
|
||||
build:darwin_x86_64 --cpu=darwin_x86_64
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
# Copyright 2021 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
#
|
||||
# This file was assembled from multiple pieces, whose use is documented
|
||||
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||
# for more information.
|
||||
|
||||
# Number of days of inactivity before an Issue or Pull Request becomes stale
|
||||
daysUntilStale: 7
|
||||
# Number of days of inactivity before a stale Issue or Pull Request is closed
|
||||
daysUntilClose: 7
|
||||
# Only issues or pull requests with all of these labels are checked if stale. Defaults to `[]` (disabled)
|
||||
onlyLabels:
|
||||
- stat:awaiting response
|
||||
# Comment to post when marking as stale. Set to `false` to disable
|
||||
markComment: >
|
||||
This issue has been automatically marked as stale because it has not had
|
||||
recent activity. It will be closed if no further activity occurs. Thank you.
|
||||
# Comment to post when removing the stale label. Set to `false` to disable
|
||||
unmarkComment: false
|
||||
closeComment: >
|
||||
Closing as stale. Please reopen if you'd like to work on this further.
|
||||
@@ -0,0 +1,66 @@
|
||||
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
# This workflow alerts and then closes the stale issues/PRs after specific time
|
||||
# You can adjust the behavior by modifying this file.
|
||||
# For more information, see:
|
||||
# https://github.com/actions/stale
|
||||
|
||||
name: 'Close stale issues and PRs'
|
||||
"on":
|
||||
schedule:
|
||||
- cron: "30 1 * * *"
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: 'actions/stale@v7'
|
||||
with:
|
||||
# Comma separated list of labels that can be assigned to issues to exclude them from being marked as stale.
|
||||
exempt-issue-labels: 'override-stale'
|
||||
# Comma separated list of labels that can be assigned to PRs to exclude them from being marked as stale.
|
||||
exempt-pr-labels: "override-stale"
|
||||
# Limit the No. of API calls in one run default value is 30.
|
||||
operations-per-run: 500
|
||||
# Prevent to remove stale label when PRs or issues are updated.
|
||||
remove-stale-when-updated: false
|
||||
# comment on issue if not active for more then 7 days.
|
||||
stale-issue-message: 'This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.'
|
||||
# comment on PR if not active for more then 14 days.
|
||||
stale-pr-message: 'This PR has been marked stale because it has no recent activity since 14 days. It will be closed if no further activity occurs. Thank you.'
|
||||
# comment on issue if stale for more then 7 days.
|
||||
close-issue-message: This issue was closed due to lack of activity after being marked stale for past 7 days.
|
||||
# comment on PR if stale for more then 14 days.
|
||||
close-pr-message: This PR was closed due to lack of activity after being marked stale for past 14 days.
|
||||
# Number of days of inactivity before an Issue Request becomes stale
|
||||
days-before-issue-stale: 7
|
||||
# Number of days of inactivity before a stale Issue is closed
|
||||
days-before-issue-close: 7
|
||||
# reason for closed the issue default value is not_planned
|
||||
close-issue-reason: completed
|
||||
# Number of days of inactivity before a stale PR is closed
|
||||
days-before-pr-close: 14
|
||||
# Number of days of inactivity before an PR Request becomes stale
|
||||
days-before-pr-stale: 14
|
||||
# Check for label to stale or close the issue/PR
|
||||
any-of-labels: 'stat:awaiting response'
|
||||
# override stale to stalled for PR
|
||||
stale-pr-label: 'stale'
|
||||
# override stale to stalled for Issue
|
||||
stale-issue-label: "stale"
|
||||
@@ -1,99 +1,121 @@
|
||||
---
|
||||
layout: default
|
||||
layout: forward
|
||||
target: https://developers.google.com/mediapipe
|
||||
title: Home
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
|
||||
|
||||
----
|
||||
|
||||
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
|
||||
**Attention:** *We have moved to
|
||||
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
|
||||
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
||||
|
||||
*This notice and web page will be removed on June 1, 2023.*
|
||||
|
||||
|
||||
----
|
||||
**Attention**: MediaPipe Solutions Preview is an early release. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/about#notice).
|
||||
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
**On-device machine learning for everyone**
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
Delight your customers with innovative machine learning features. MediaPipe
|
||||
contains everything that you need to customize and deploy to mobile (Android,
|
||||
iOS), web, desktop, edge devices, and IoT, effortlessly.
|
||||
|
||||
## Live ML anywhere
|
||||
* [See demos](https://goo.gle/mediapipe-studio)
|
||||
* [Learn more](https://developers.google.com/mediapipe/solutions)
|
||||
|
||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
||||
ML solutions for live and streaming media.
|
||||
## Get started
|
||||
|
||||
 | 
|
||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
 | 
|
||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
||||
You can get started with MediaPipe Solutions by by checking out any of the
|
||||
developer guides for
|
||||
[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
|
||||
[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
|
||||
and
|
||||
[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
|
||||
tasks. If you need help setting up a development environment for use with
|
||||
MediaPipe Tasks, check out the setup guides for
|
||||
[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
|
||||
apps](https://developers.google.com/mediapipe/solutions/setup_web), and
|
||||
[Python](https://developers.google.com/mediapipe/solutions/setup_python).
|
||||
|
||||
----
|
||||
## Solutions
|
||||
|
||||
## ML solutions in MediaPipe
|
||||
MediaPipe Solutions provides a suite of libraries and tools for you to quickly
|
||||
apply artificial intelligence (AI) and machine learning (ML) techniques in your
|
||||
applications. You can plug these solutions into your applications immediately,
|
||||
customize them to your needs, and use them across multiple development
|
||||
platforms. MediaPipe Solutions is part of the MediaPipe [open source
|
||||
project](https://github.com/google/mediapipe), so you can further customize the
|
||||
solutions code to meet your application needs.
|
||||
|
||||
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
|
||||
These libraries and resources provide the core functionality for each MediaPipe
|
||||
Solution:
|
||||
|
||||
Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
||||
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
|
||||
solutions. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/tasks).
|
||||
* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
|
||||
solution.
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
||||
These tools let you customize and evaluate solutions:
|
||||
|
||||
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
||||
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
* **MediaPipe Model Maker**: Customize models for solutions with your data.
|
||||
[Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
|
||||
* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
|
||||
browser. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/studio).
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
### Legacy solutions
|
||||
|
||||
## Getting started
|
||||
We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
|
||||
a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
|
||||
and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
|
||||
provided on an as-is basis.
|
||||
|
||||
To start using MediaPipe
|
||||
[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
|
||||
lines code, see example code and demos in
|
||||
[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
|
||||
[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
|
||||
For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
|
||||
|
||||
To use MediaPipe in C++, Android and iOS, which allow further customization of
|
||||
the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
|
||||
building your own, learn how to
|
||||
[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
|
||||
start building example applications in
|
||||
[C++](https://google.github.io/mediapipe/getting_started/cpp),
|
||||
[Android](https://google.github.io/mediapipe/getting_started/android) and
|
||||
[iOS](https://google.github.io/mediapipe/getting_started/ios).
|
||||
## Framework
|
||||
|
||||
The source code is hosted in the
|
||||
[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
|
||||
run code search using
|
||||
[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
|
||||
To start using MediaPipe Framework, [install MediaPipe
|
||||
Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
|
||||
and start building example applications in C++, Android, and iOS.
|
||||
|
||||
## Publications
|
||||
[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
|
||||
low-level component used to build efficient on-device machine learning
|
||||
pipelines, similar to the premade MediaPipe Solutions.
|
||||
|
||||
Before using MediaPipe Framework, familiarize yourself with the following key
|
||||
[Framework
|
||||
concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
|
||||
|
||||
* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
|
||||
* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
|
||||
* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
|
||||
|
||||
## Community
|
||||
|
||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
|
||||
users.
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||
community discussion around MediaPipe.
|
||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||
curated list of awesome MediaPipe related frameworks, libraries and
|
||||
software.
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions. Please follow these
|
||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||
|
||||
## Resources
|
||||
|
||||
### Publications
|
||||
|
||||
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
||||
in Google Developers Blog
|
||||
@@ -102,7 +124,8 @@ run code search using
|
||||
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
||||
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
||||
in Google Developers Blog
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
|
||||
Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
in Google AI Blog
|
||||
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
||||
in Google AI Blog
|
||||
@@ -130,43 +153,6 @@ run code search using
|
||||
in Google AI Blog
|
||||
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
||||
|
||||
## Videos
|
||||
### Videos
|
||||
|
||||
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
|
||||
|
||||
## Events
|
||||
|
||||
* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
|
||||
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
|
||||
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
|
||||
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
|
||||
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
|
||||
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
|
||||
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
|
||||
Seoul, Korea ICCV
|
||||
2019](https://research.google.com/youtube8m/workshop2019/index.html)
|
||||
* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
|
||||
* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
|
||||
([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
|
||||
* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
|
||||
|
||||
## Community
|
||||
|
||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||
curated list of awesome MediaPipe related frameworks, libraries and software
|
||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||
community discussion around MediaPipe
|
||||
|
||||
## Alpha disclaimer
|
||||
|
||||
MediaPipe is currently in alpha at v0.7. We may be still making breaking API
|
||||
changes and expect to get to stable APIs by v1.0.
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions. Please follow these
|
||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||
|
||||
@@ -266,10 +266,10 @@ http_archive(
|
||||
|
||||
http_archive(
|
||||
name = "com_googlesource_code_re2",
|
||||
sha256 = "e06b718c129f4019d6e7aa8b7631bee38d3d450dd980246bfaf493eb7db67868",
|
||||
strip_prefix = "re2-fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8",
|
||||
sha256 = "ef516fb84824a597c4d5d0d6d330daedb18363b5a99eda87d027e6bdd9cba299",
|
||||
strip_prefix = "re2-03da4fc0857c285e3a26782f6bc8931c4c950df4",
|
||||
urls = [
|
||||
"https://github.com/google/re2/archive/fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8.tar.gz",
|
||||
"https://github.com/google/re2/archive/03da4fc0857c285e3a26782f6bc8931c4c950df4.tar.gz",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -375,6 +375,22 @@ http_archive(
|
||||
url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
|
||||
)
|
||||
|
||||
# Building an opencv.xcframework from the OpenCV 4.5.3 sources is necessary for
|
||||
# MediaPipe iOS Task Libraries to be supported on arm64(M1) Macs. An
|
||||
# `opencv.xcframework` archive has not been released and it is recommended to
|
||||
# build the same from source using a script provided in OpenCV 4.5.0 upwards.
|
||||
# OpenCV is fixed to version to 4.5.3 since swift support can only be disabled
|
||||
# from 4.5.3 upwards. This is needed to avoid errors when the library is linked
|
||||
# in Xcode. Swift support will be added in when the final binary MediaPipe iOS
|
||||
# Task libraries are built.
|
||||
http_archive(
|
||||
name = "ios_opencv_source",
|
||||
sha256 = "a61e7a4618d353140c857f25843f39b2abe5f451b018aab1604ef0bc34cd23d5",
|
||||
build_file = "@//third_party:opencv_ios_source.BUILD",
|
||||
type = "zip",
|
||||
url = "https://github.com/opencv/opencv/archive/refs/tags/4.5.3.zip",
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "stblib",
|
||||
strip_prefix = "stb-b42009b3b9d4ca35bc703f5310eedc74f584be58",
|
||||
@@ -468,9 +484,9 @@ http_archive(
|
||||
)
|
||||
|
||||
# TensorFlow repo should always go after the other external dependencies.
|
||||
# TF on 2023-04-12.
|
||||
_TENSORFLOW_GIT_COMMIT = "d712c0c9e24519cc8cd3720279666720d1000eee"
|
||||
_TENSORFLOW_SHA256 = "ba98de6ea5f720071246691a1536ecd5e1b1763033e8c82a1e721a06d3dfd4c1"
|
||||
# TF on 2023-05-26.
|
||||
_TENSORFLOW_GIT_COMMIT = "67d5c561981edc45daf3f9d73ddd1a77963733ca"
|
||||
_TENSORFLOW_SHA256 = "0c8326285e9cb695313e194b97d388eea70bf8bf5b13e8f0962ca8eed5179ece"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
urls = [
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
@@ -14,6 +14,7 @@
|
||||
# ==============================================================================
|
||||
"""Generate Java reference docs for MediaPipe."""
|
||||
import pathlib
|
||||
import shutil
|
||||
|
||||
from absl import app
|
||||
from absl import flags
|
||||
@@ -41,7 +42,9 @@ def main(_) -> None:
|
||||
mp_root = pathlib.Path(__file__)
|
||||
while (mp_root := mp_root.parent).name != 'mediapipe':
|
||||
# Find the nearest `mediapipe` dir.
|
||||
pass
|
||||
if not mp_root.name:
|
||||
# We've hit the filesystem root - abort.
|
||||
raise FileNotFoundError('"mediapipe" root not found')
|
||||
|
||||
# Find the root from which all packages are relative.
|
||||
root = mp_root.parent
|
||||
@@ -51,6 +54,14 @@ def main(_) -> None:
|
||||
if (mp_root / 'mediapipe').exists():
|
||||
mp_root = mp_root / 'mediapipe'
|
||||
|
||||
# We need to copy this into the tasks dir to ensure we don't leave broken
|
||||
# links in the generated docs.
|
||||
old_api_dir = 'java/com/google/mediapipe/framework/image'
|
||||
shutil.copytree(
|
||||
mp_root / old_api_dir,
|
||||
mp_root / 'tasks' / old_api_dir,
|
||||
dirs_exist_ok=True)
|
||||
|
||||
gen_java.gen_java_docs(
|
||||
package='com.google.mediapipe',
|
||||
source_path=mp_root / 'tasks/java',
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
+93
-107
@@ -1,99 +1,121 @@
|
||||
---
|
||||
layout: default
|
||||
layout: forward
|
||||
target: https://developers.google.com/mediapipe
|
||||
title: Home
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
|
||||
|
||||
----
|
||||
|
||||
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
|
||||
**Attention:** *We have moved to
|
||||
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
|
||||
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
||||
|
||||
*This notice and web page will be removed on June 1, 2023.*
|
||||
|
||||
|
||||
----
|
||||
**Attention**: MediaPipe Solutions Preview is an early release. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/about#notice).
|
||||
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
**On-device machine learning for everyone**
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
Delight your customers with innovative machine learning features. MediaPipe
|
||||
contains everything that you need to customize and deploy to mobile (Android,
|
||||
iOS), web, desktop, edge devices, and IoT, effortlessly.
|
||||
|
||||
## Live ML anywhere
|
||||
* [See demos](https://goo.gle/mediapipe-studio)
|
||||
* [Learn more](https://developers.google.com/mediapipe/solutions)
|
||||
|
||||
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
|
||||
ML solutions for live and streaming media.
|
||||
## Get started
|
||||
|
||||
 | 
|
||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
||||
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
 | 
|
||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
||||
You can get started with MediaPipe Solutions by by checking out any of the
|
||||
developer guides for
|
||||
[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
|
||||
[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
|
||||
and
|
||||
[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
|
||||
tasks. If you need help setting up a development environment for use with
|
||||
MediaPipe Tasks, check out the setup guides for
|
||||
[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
|
||||
apps](https://developers.google.com/mediapipe/solutions/setup_web), and
|
||||
[Python](https://developers.google.com/mediapipe/solutions/setup_python).
|
||||
|
||||
----
|
||||
## Solutions
|
||||
|
||||
## ML solutions in MediaPipe
|
||||
MediaPipe Solutions provides a suite of libraries and tools for you to quickly
|
||||
apply artificial intelligence (AI) and machine learning (ML) techniques in your
|
||||
applications. You can plug these solutions into your applications immediately,
|
||||
customize them to your needs, and use them across multiple development
|
||||
platforms. MediaPipe Solutions is part of the MediaPipe [open source
|
||||
project](https://github.com/google/mediapipe), so you can further customize the
|
||||
solutions code to meet your application needs.
|
||||
|
||||
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
|
||||
These libraries and resources provide the core functionality for each MediaPipe
|
||||
Solution:
|
||||
|
||||
Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
|
||||
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
|
||||
solutions. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/tasks).
|
||||
* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
|
||||
solution.
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
||||
These tools let you customize and evaluate solutions:
|
||||
|
||||
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
||||
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
* **MediaPipe Model Maker**: Customize models for solutions with your data.
|
||||
[Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
|
||||
* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
|
||||
browser. [Learn
|
||||
more](https://developers.google.com/mediapipe/solutions/studio).
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
### Legacy solutions
|
||||
|
||||
## Getting started
|
||||
We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
|
||||
a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
|
||||
and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
|
||||
provided on an as-is basis.
|
||||
|
||||
To start using MediaPipe
|
||||
[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
|
||||
lines code, see example code and demos in
|
||||
[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
|
||||
[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
|
||||
For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
|
||||
|
||||
To use MediaPipe in C++, Android and iOS, which allow further customization of
|
||||
the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
|
||||
building your own, learn how to
|
||||
[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
|
||||
start building example applications in
|
||||
[C++](https://google.github.io/mediapipe/getting_started/cpp),
|
||||
[Android](https://google.github.io/mediapipe/getting_started/android) and
|
||||
[iOS](https://google.github.io/mediapipe/getting_started/ios).
|
||||
## Framework
|
||||
|
||||
The source code is hosted in the
|
||||
[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
|
||||
run code search using
|
||||
[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
|
||||
To start using MediaPipe Framework, [install MediaPipe
|
||||
Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
|
||||
and start building example applications in C++, Android, and iOS.
|
||||
|
||||
## Publications
|
||||
[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
|
||||
low-level component used to build efficient on-device machine learning
|
||||
pipelines, similar to the premade MediaPipe Solutions.
|
||||
|
||||
Before using MediaPipe Framework, familiarize yourself with the following key
|
||||
[Framework
|
||||
concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
|
||||
|
||||
* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
|
||||
* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
|
||||
* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
|
||||
|
||||
## Community
|
||||
|
||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
|
||||
users.
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||
community discussion around MediaPipe.
|
||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||
curated list of awesome MediaPipe related frameworks, libraries and
|
||||
software.
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions. Please follow these
|
||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||
|
||||
## Resources
|
||||
|
||||
### Publications
|
||||
|
||||
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
|
||||
in Google Developers Blog
|
||||
@@ -102,7 +124,8 @@ run code search using
|
||||
* [SignAll SDK: Sign language interface using MediaPipe is now available for
|
||||
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
|
||||
in Google Developers Blog
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
|
||||
Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
in Google AI Blog
|
||||
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
|
||||
in Google AI Blog
|
||||
@@ -130,43 +153,6 @@ run code search using
|
||||
in Google AI Blog
|
||||
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
||||
|
||||
## Videos
|
||||
### Videos
|
||||
|
||||
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
|
||||
|
||||
## Events
|
||||
|
||||
* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
|
||||
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
|
||||
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
|
||||
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
|
||||
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
|
||||
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
|
||||
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
|
||||
Seoul, Korea ICCV
|
||||
2019](https://research.google.com/youtube8m/workshop2019/index.html)
|
||||
* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
|
||||
* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
|
||||
([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
|
||||
* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
|
||||
|
||||
## Community
|
||||
|
||||
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
|
||||
curated list of awesome MediaPipe related frameworks, libraries and software
|
||||
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||
community discussion around MediaPipe
|
||||
|
||||
## Alpha disclaimer
|
||||
|
||||
MediaPipe is currently in alpha at v0.7. We may be still making breaking API
|
||||
changes and expect to get to stable APIs by v1.0.
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions. Please follow these
|
||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||
|
||||
@@ -20,9 +20,9 @@ nav_order: 1
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_detector)
|
||||
site.*
|
||||
|
||||
----
|
||||
|
||||
@@ -20,9 +20,9 @@ nav_order: 2
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
|
||||
site.*
|
||||
|
||||
----
|
||||
|
||||
@@ -20,9 +20,9 @@ nav_order: 3
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
|
||||
site.*
|
||||
|
||||
----
|
||||
|
||||
@@ -22,9 +22,9 @@ nav_order: 5
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker)
|
||||
site.*
|
||||
|
||||
----
|
||||
|
||||
@@ -21,7 +21,7 @@ nav_order: 1
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions.
|
||||
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
|
||||
As of May 10, 2023, this solution was upgraded to a new MediaPipe
|
||||
Solution. For more information, see the
|
||||
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
|
||||
site.*
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
---
|
||||
layout: default
|
||||
layout: forward
|
||||
target: https://developers.google.com/mediapipe/solutions/guide#legacy
|
||||
title: MediaPipe Legacy Solutions
|
||||
nav_order: 3
|
||||
has_children: true
|
||||
@@ -13,8 +14,7 @@ has_toc: false
|
||||
{:toc}
|
||||
---
|
||||
|
||||
**Attention:** *Thank you for your interest in MediaPipe Solutions. We have
|
||||
ended support for
|
||||
**Attention:** *We have ended support for
|
||||
[these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
as of March 1, 2023. All other
|
||||
[MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
|
||||
@@ -25,14 +25,6 @@ be provided on an as-is basis. We encourage you to check out the new MediaPipe
|
||||
Solutions at:
|
||||
[https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
|
||||
|
||||
*This notice and web page will be removed on June 1, 2023.*
|
||||
|
||||
----
|
||||
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
<br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
----
|
||||
|
||||
MediaPipe offers open source cross-platform, customizable ML solutions for live
|
||||
|
||||
@@ -141,6 +141,7 @@ config_setting(
|
||||
"ios_armv7",
|
||||
"ios_arm64",
|
||||
"ios_arm64e",
|
||||
"ios_sim_arm64",
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
@@ -433,9 +433,9 @@ absl::Status SpectrogramCalculator::ProcessVectorToOutput(
|
||||
absl::Status SpectrogramCalculator::ProcessVector(const Matrix& input_stream,
|
||||
CalculatorContext* cc) {
|
||||
switch (output_type_) {
|
||||
// These blocks deliberately ignore clang-format to preserve the
|
||||
// "silhouette" of the different cases.
|
||||
// clang-format off
|
||||
// These blocks deliberately ignore clang-format to preserve the
|
||||
// "silhouette" of the different cases.
|
||||
// clang-format off
|
||||
case SpectrogramCalculatorOptions::COMPLEX: {
|
||||
return ProcessVectorToOutput(
|
||||
input_stream,
|
||||
|
||||
@@ -192,17 +192,19 @@ cc_library(
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -215,18 +217,18 @@ cc_library(
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:collection_item_id",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/util:render_data_cc_proto",
|
||||
"@com_google_absl//absl/status",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
],
|
||||
alwayslink = 1,
|
||||
@@ -295,8 +297,7 @@ cc_library(
|
||||
"//mediapipe/util:render_data_cc_proto",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//mediapipe:ios": [],
|
||||
":ios_or_disable_gpu": [],
|
||||
"//conditions:default": [
|
||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||
],
|
||||
@@ -904,6 +905,7 @@ cc_library(
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
@@ -1238,6 +1240,7 @@ cc_library(
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
|
||||
@@ -17,10 +17,13 @@
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
@@ -60,4 +63,17 @@ REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
|
||||
typedef BeginLoopCalculator<std::vector<Tensor>> BeginLoopTensorCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopTensorCalculator);
|
||||
|
||||
// A calculator to process std::vector<mediapipe::ImageFrame>.
|
||||
typedef BeginLoopCalculator<std::vector<ImageFrame>>
|
||||
BeginLoopImageFrameCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopImageFrameCalculator);
|
||||
|
||||
// A calculator to process std::vector<mediapipe::GpuBuffer>.
|
||||
typedef BeginLoopCalculator<std::vector<GpuBuffer>>
|
||||
BeginLoopGpuBufferCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator);
|
||||
|
||||
// A calculator to process std::vector<mediapipe::Image>.
|
||||
typedef BeginLoopCalculator<std::vector<Image>> BeginLoopImageCalculator;
|
||||
REGISTER_CALCULATOR(BeginLoopImageCalculator);
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -15,47 +15,57 @@
|
||||
#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Calculator for implementing loops on iterable collections inside a MediaPipe
|
||||
// graph.
|
||||
// graph. Assume InputIterT is an iterable for type InputT, and OutputIterT is
|
||||
// an iterable for type OutputT, e.g. vector<InputT> and vector<OutputT>.
|
||||
// First, instantiate specializations in the loop calculators' implementations
|
||||
// if missing:
|
||||
// BeginLoopInputTCalculator = BeginLoopCalculator<InputIterT>
|
||||
// EndLoopOutputTCalculator = EndLoopCalculator<OutputIterT>
|
||||
// Then, the following graph transforms an item of type InputIterT to an
|
||||
// OutputIterT by applying InputToOutputConverter to every element:
|
||||
//
|
||||
// It is designed to be used like:
|
||||
//
|
||||
// node {
|
||||
// calculator: "BeginLoopWithIterableCalculator"
|
||||
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
|
||||
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
|
||||
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// node { # Type @timestamp
|
||||
// calculator: "BeginLoopInputTCalculator"
|
||||
// input_stream: "ITERABLE:input_iterable" # InputIterT @iterable_ts
|
||||
// input_stream: "CLONE:extra_input" # ExtraT @extra_ts
|
||||
// output_stream: "ITEM:input_iterator" # InputT @loop_internal_ts
|
||||
// output_stream: "CLONE:cloned_extra_input" # ExtraT @loop_internal_ts
|
||||
// output_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "ElementToBlaConverterSubgraph"
|
||||
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
|
||||
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// calculator: "InputToOutputConverter"
|
||||
// input_stream: "INPUT:input_iterator" # InputT @loop_internal_ts
|
||||
// input_stream: "EXTRA:cloned_extra_input" # ExtraT @loop_internal_ts
|
||||
// output_stream: "OUTPUT:output_iterator" # OutputT @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "EndLoopWithOutputCalculator"
|
||||
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
|
||||
// calculator: "EndLoopOutputTCalculator"
|
||||
// input_stream: "ITEM:output_iterator" # OutputT @loop_internal_ts
|
||||
// input_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
|
||||
// output_stream: "ITERABLE:output_iterable" # OutputIterT @iterable_ts
|
||||
// }
|
||||
//
|
||||
// The resulting 'output_iterable' has the same timestamp as 'input_iterable'.
|
||||
// The output packets of this calculator are part of the loop body and have
|
||||
// loop-internal timestamps that are unrelated to the input iterator timestamp.
|
||||
//
|
||||
// Input streams tagged with "CLONE" are cloned to the corresponding output
|
||||
// streams at loop timestamps. This ensures that a MediaPipe graph or sub-graph
|
||||
// can run multiple times, once per element in the "ITERABLE" for each pakcet
|
||||
// clone of the packets in the "CLONE" input streams.
|
||||
// streams at loop-internal timestamps. This ensures that a MediaPipe graph or
|
||||
// sub-graph can run multiple times, once per element in the "ITERABLE" for each
|
||||
// packet clone of the packets in the "CLONE" input streams. Think of CLONEd
|
||||
// inputs as loop-wide constants.
|
||||
template <typename IterableT>
|
||||
class BeginLoopCalculator : public CalculatorBase {
|
||||
using ItemT = typename IterableT::value_type;
|
||||
|
||||
@@ -55,6 +55,10 @@ MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator);
|
||||
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator);
|
||||
|
||||
typedef ConcatenateVectorCalculator<std::string>
|
||||
ConcatenateStringVectorCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(ConcatenateStringVectorCalculator);
|
||||
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
|
||||
|
||||
@@ -30,13 +30,15 @@ namespace mediapipe {
|
||||
typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
|
||||
|
||||
void AddInputVector(int index, const std::vector<int>& input, int64_t timestamp,
|
||||
template <typename T>
|
||||
void AddInputVector(int index, const std::vector<T>& input, int64_t timestamp,
|
||||
CalculatorRunner* runner) {
|
||||
runner->MutableInputs()->Index(index).packets.push_back(
|
||||
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
|
||||
MakePacket<std::vector<T>>(input).At(Timestamp(timestamp)));
|
||||
}
|
||||
|
||||
void AddInputVectors(const std::vector<std::vector<int>>& inputs,
|
||||
template <typename T>
|
||||
void AddInputVectors(const std::vector<std::vector<T>>& inputs,
|
||||
int64_t timestamp, CalculatorRunner* runner) {
|
||||
for (int i = 0; i < inputs.size(); ++i) {
|
||||
AddInputVector(i, inputs[i], timestamp, runner);
|
||||
@@ -382,6 +384,23 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
|
||||
EXPECT_EQ(0, outputs.size());
|
||||
}
|
||||
|
||||
TEST(ConcatenateStringVectorCalculatorTest, OneTimestamp) {
|
||||
CalculatorRunner runner("ConcatenateStringVectorCalculator",
|
||||
/*options_string=*/"", /*num_inputs=*/3,
|
||||
/*num_outputs=*/1, /*num_side_packets=*/0);
|
||||
|
||||
std::vector<std::vector<std::string>> inputs = {
|
||||
{"a", "b"}, {"c"}, {"d", "e", "f"}};
|
||||
AddInputVectors(inputs, /*timestamp=*/1, &runner);
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
|
||||
EXPECT_EQ(1, outputs.size());
|
||||
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
|
||||
std::vector<std::string> expected_vector = {"a", "b", "c", "d", "e", "f"};
|
||||
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<std::string>>());
|
||||
}
|
||||
|
||||
typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
|
||||
TestConcatenateUniqueIntPtrCalculator;
|
||||
MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator);
|
||||
|
||||
@@ -19,10 +19,12 @@
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "mediapipe/util/render_data.pb.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
@@ -68,8 +70,18 @@ REGISTER_CALCULATOR(EndLoopMatrixCalculator);
|
||||
typedef EndLoopCalculator<std::vector<Tensor>> EndLoopTensorCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopTensorCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<ImageFrame>> EndLoopImageFrameCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopImageFrameCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<GpuBuffer>> EndLoopGpuBufferCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopGpuBufferCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<::mediapipe::Image>>
|
||||
EndLoopImageCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopImageCalculator);
|
||||
|
||||
typedef EndLoopCalculator<std::vector<std::array<float, 16>>>
|
||||
EndLoopAffineMatrixCalculator;
|
||||
REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -17,13 +17,11 @@
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/collection_item_id.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
@@ -33,27 +31,7 @@ namespace mediapipe {
|
||||
// from the "BATCH_END" tagged input stream, it emits the aggregated results
|
||||
// at the original timestamp contained in the "BATCH_END" input stream.
|
||||
//
|
||||
// It is designed to be used like:
|
||||
//
|
||||
// node {
|
||||
// calculator: "BeginLoopWithIterableCalculator"
|
||||
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
|
||||
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
|
||||
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "ElementToBlaConverterSubgraph"
|
||||
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
|
||||
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "EndLoopWithOutputCalculator"
|
||||
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
|
||||
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
|
||||
// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
|
||||
// }
|
||||
// See BeginLoopCalculator for a usage example.
|
||||
template <typename IterableT>
|
||||
class EndLoopCalculator : public CalculatorBase {
|
||||
using ItemT = typename IterableT::value_type;
|
||||
@@ -79,7 +57,7 @@ class EndLoopCalculator : public CalculatorBase {
|
||||
}
|
||||
// Try to consume the item and move it into the collection. If the items
|
||||
// are not consumable, then try to copy them instead. If the items are
|
||||
// not copiable, then an error will be returned.
|
||||
// not copyable, then an error will be returned.
|
||||
auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
|
||||
if (item_ptr_or.ok()) {
|
||||
input_stream_collection_->push_back(std::move(*item_ptr_or.value()));
|
||||
|
||||
@@ -42,7 +42,7 @@ constexpr char kOptionsTag[] = "OPTIONS";
|
||||
//
|
||||
// Increasing `max_in_flight` to 2 or more can yield the better throughput
|
||||
// when the graph exhibits a high degree of pipeline parallelism. Decreasing
|
||||
// `max_in_flight` to 0 can yield a better average latency, but at the cost of
|
||||
// `max_in_queue` to 0 can yield a better average latency, but at the cost of
|
||||
// lower throughput (lower framerate) due to the time during which the graph
|
||||
// is idle awaiting the next input frame.
|
||||
//
|
||||
|
||||
@@ -26,19 +26,15 @@ constexpr char kStateChangeTag[] = "STATE_CHANGE";
|
||||
constexpr char kDisallowTag[] = "DISALLOW";
|
||||
constexpr char kAllowTag[] = "ALLOW";
|
||||
|
||||
enum GateState {
|
||||
GATE_UNINITIALIZED,
|
||||
GATE_ALLOW,
|
||||
GATE_DISALLOW,
|
||||
};
|
||||
|
||||
std::string ToString(GateState state) {
|
||||
std::string ToString(GateCalculatorOptions::GateState state) {
|
||||
switch (state) {
|
||||
case GATE_UNINITIALIZED:
|
||||
case GateCalculatorOptions::UNSPECIFIED:
|
||||
return "UNSPECIFIED";
|
||||
case GateCalculatorOptions::GATE_UNINITIALIZED:
|
||||
return "UNINITIALIZED";
|
||||
case GATE_ALLOW:
|
||||
case GateCalculatorOptions::GATE_ALLOW:
|
||||
return "ALLOW";
|
||||
case GATE_DISALLOW:
|
||||
case GateCalculatorOptions::GATE_DISALLOW:
|
||||
return "DISALLOW";
|
||||
}
|
||||
DLOG(FATAL) << "Unknown GateState";
|
||||
@@ -153,10 +149,12 @@ class GateCalculator : public CalculatorBase {
|
||||
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
num_data_streams_ = cc->Inputs().NumEntries("");
|
||||
last_gate_state_ = GATE_UNINITIALIZED;
|
||||
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
|
||||
|
||||
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
|
||||
last_gate_state_ = options.initial_gate_state();
|
||||
|
||||
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
|
||||
|
||||
empty_packets_as_allow_ = options.empty_packets_as_allow();
|
||||
|
||||
if (!use_side_packet_for_allow_disallow_ &&
|
||||
@@ -184,10 +182,12 @@ class GateCalculator : public CalculatorBase {
|
||||
allow = !cc->Inputs().Tag(kDisallowTag).Get<bool>();
|
||||
}
|
||||
}
|
||||
const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
|
||||
const GateCalculatorOptions::GateState new_gate_state =
|
||||
allow ? GateCalculatorOptions::GATE_ALLOW
|
||||
: GateCalculatorOptions::GATE_DISALLOW;
|
||||
|
||||
if (cc->Outputs().HasTag(kStateChangeTag)) {
|
||||
if (last_gate_state_ != GATE_UNINITIALIZED &&
|
||||
if (last_gate_state_ != GateCalculatorOptions::GATE_UNINITIALIZED &&
|
||||
last_gate_state_ != new_gate_state) {
|
||||
VLOG(2) << "State transition in " << cc->NodeName() << " @ "
|
||||
<< cc->InputTimestamp().Value() << " from "
|
||||
@@ -223,7 +223,8 @@ class GateCalculator : public CalculatorBase {
|
||||
}
|
||||
|
||||
private:
|
||||
GateState last_gate_state_ = GATE_UNINITIALIZED;
|
||||
GateCalculatorOptions::GateState last_gate_state_ =
|
||||
GateCalculatorOptions::GATE_UNINITIALIZED;
|
||||
int num_data_streams_;
|
||||
bool empty_packets_as_allow_;
|
||||
bool use_side_packet_for_allow_disallow_ = false;
|
||||
|
||||
@@ -31,4 +31,13 @@ message GateCalculatorOptions {
|
||||
// Whether to allow or disallow the input streams to pass when no
|
||||
// ALLOW/DISALLOW input or side input is specified.
|
||||
optional bool allow = 2 [default = false];
|
||||
|
||||
enum GateState {
|
||||
UNSPECIFIED = 0;
|
||||
GATE_UNINITIALIZED = 1;
|
||||
GATE_ALLOW = 2;
|
||||
GATE_DISALLOW = 3;
|
||||
}
|
||||
|
||||
optional GateState initial_gate_state = 3 [default = GATE_UNINITIALIZED];
|
||||
}
|
||||
|
||||
@@ -458,5 +458,29 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
|
||||
ASSERT_EQ(0, output.size());
|
||||
}
|
||||
|
||||
// Must detect allow value for first timestamp as a state change when the
|
||||
// initial state is set to GATE_DISALLOW.
|
||||
TEST_F(GateCalculatorTest, StateChangeTriggeredWithInitialGateStateOption) {
|
||||
SetRunner(R"(
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "test_input"
|
||||
input_stream: "ALLOW:allow"
|
||||
output_stream: "test_output"
|
||||
output_stream: "STATE_CHANGE:state_change"
|
||||
options: {
|
||||
[mediapipe.GateCalculatorOptions.ext] {
|
||||
initial_gate_state: GATE_DISALLOW
|
||||
}
|
||||
}
|
||||
)");
|
||||
|
||||
constexpr int64_t kTimestampValue0 = 42;
|
||||
RunTimeStep(kTimestampValue0, "ALLOW", true);
|
||||
|
||||
const std::vector<Packet>& output =
|
||||
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
||||
ASSERT_EQ(1, output.size());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
@@ -37,5 +38,12 @@ using GetDetectionVectorItemCalculator =
|
||||
GetVectorItemCalculator<mediapipe::Detection>;
|
||||
REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
|
||||
|
||||
using GetNormalizedRectVectorItemCalculator =
|
||||
GetVectorItemCalculator<NormalizedRect>;
|
||||
REGISTER_CALCULATOR(GetNormalizedRectVectorItemCalculator);
|
||||
|
||||
using GetRectVectorItemCalculator = GetVectorItemCalculator<Rect>;
|
||||
REGISTER_CALCULATOR(GetRectVectorItemCalculator);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -35,7 +35,7 @@ class MatrixToVectorCalculatorTest
|
||||
void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
|
||||
|
||||
void AppendInput(const std::vector<float>& column_major_data,
|
||||
int64 timestamp) {
|
||||
int64_t timestamp) {
|
||||
ASSERT_EQ(num_input_samples_ * num_input_channels_,
|
||||
column_major_data.size());
|
||||
Eigen::Map<const Matrix> data_map(&column_major_data[0],
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -51,9 +51,9 @@ class SimpleRunner : public CalculatorRunner {
|
||||
|
||||
virtual ~SimpleRunner() {}
|
||||
|
||||
void SetInput(const std::vector<int64>& timestamp_list) {
|
||||
void SetInput(const std::vector<int64_t>& timestamp_list) {
|
||||
MutableInputs()->Index(0).packets.clear();
|
||||
for (const int64 ts : timestamp_list) {
|
||||
for (const int64_t ts : timestamp_list) {
|
||||
MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(new std::string(absl::StrCat("Frame #", ts)))
|
||||
.At(Timestamp(ts)));
|
||||
@@ -72,8 +72,8 @@ class SimpleRunner : public CalculatorRunner {
|
||||
}
|
||||
|
||||
void CheckOutputTimestamps(
|
||||
const std::vector<int64>& expected_frames,
|
||||
const std::vector<int64>& expected_timestamps) const {
|
||||
const std::vector<int64_t>& expected_frames,
|
||||
const std::vector<int64_t>& expected_timestamps) const {
|
||||
EXPECT_EQ(expected_frames.size(), Outputs().Index(0).packets.size());
|
||||
EXPECT_EQ(expected_timestamps.size(), Outputs().Index(0).packets.size());
|
||||
int count = 0;
|
||||
@@ -112,7 +112,7 @@ MATCHER_P2(PacketAtTimestamp, payload, timestamp,
|
||||
*result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
|
||||
return false;
|
||||
}
|
||||
int64 actual_payload = arg.template Get<int64>();
|
||||
int64_t actual_payload = arg.template Get<int64_t>();
|
||||
if (actual_payload != payload) {
|
||||
*result_listener << "with incorrect payload = " << actual_payload;
|
||||
return false;
|
||||
@@ -137,18 +137,18 @@ class ReproducibleJitterWithReflectionStrategyForTesting
|
||||
//
|
||||
// An EXPECT will fail if sequence is less than the number requested during
|
||||
// processing.
|
||||
static std::vector<uint64> random_sequence;
|
||||
static std::vector<uint64_t> random_sequence;
|
||||
|
||||
protected:
|
||||
virtual uint64 GetNextRandom(uint64 n) {
|
||||
virtual uint64_t GetNextRandom(uint64_t n) {
|
||||
EXPECT_LT(sequence_index_, random_sequence.size());
|
||||
return random_sequence[sequence_index_++] % n;
|
||||
}
|
||||
|
||||
private:
|
||||
int32 sequence_index_ = 0;
|
||||
int32_t sequence_index_ = 0;
|
||||
};
|
||||
std::vector<uint64>
|
||||
std::vector<uint64_t>
|
||||
ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
|
||||
|
||||
// PacketResamplerCalculator child class which injects a specified stream
|
||||
@@ -469,7 +469,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
|
||||
}
|
||||
)pb"));
|
||||
|
||||
for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
|
||||
for (const int64_t ts : {0, 5000, 10010, 15001, 19990}) {
|
||||
runner.MutableInputs()->Tag(kDataTag).packets.push_back(
|
||||
Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts)));
|
||||
}
|
||||
|
||||
@@ -43,8 +43,8 @@ constexpr char kDisallowTag[] = "DISALLOW";
|
||||
|
||||
// Returns the timestamp values for a vector of Packets.
|
||||
// TODO: puth this kind of test util in a common place.
|
||||
std::vector<int64> TimestampValues(const std::vector<Packet>& packets) {
|
||||
std::vector<int64> result;
|
||||
std::vector<int64_t> TimestampValues(const std::vector<Packet>& packets) {
|
||||
std::vector<int64_t> result;
|
||||
for (const Packet& packet : packets) {
|
||||
result.push_back(packet.Timestamp().Value());
|
||||
}
|
||||
@@ -371,7 +371,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
|
||||
for (int main_ts = 0; main_ts < 50; ++main_ts) {
|
||||
send_packet("in", main_ts);
|
||||
MP_EXPECT_OK(graph_.WaitUntilIdle());
|
||||
std::vector<int64> ts_values = TimestampValues(outputs);
|
||||
std::vector<int64_t> ts_values = TimestampValues(outputs);
|
||||
EXPECT_EQ(ts_values.size(), main_ts + 1);
|
||||
for (int j = 0; j < main_ts + 1; ++j) {
|
||||
EXPECT_EQ(ts_values[j], j);
|
||||
|
||||
@@ -121,7 +121,7 @@ absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
|
||||
if (cc->Outputs().HasTag(kTagAtTimestamp)) {
|
||||
RET_CHECK_EQ(num_entries + 1, cc->InputSidePackets().NumEntries())
|
||||
<< "For AT_TIMESTAMP tag, 2 input side packets are required.";
|
||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set<int64>();
|
||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set<int64_t>();
|
||||
} else {
|
||||
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
|
||||
<< "Same number of input side packets and output streams is required.";
|
||||
@@ -178,8 +178,8 @@ absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
|
||||
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
||||
}
|
||||
} else if (cc->Outputs().HasTag(kTagAtTimestamp)) {
|
||||
int64 timestamp =
|
||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get<int64>();
|
||||
int64_t timestamp =
|
||||
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get<int64_t>();
|
||||
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
||||
cc->Outputs()
|
||||
.Get(output_tag_, i)
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
@@ -86,4 +87,12 @@ REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
|
||||
typedef SplitVectorCalculator<float, false> SplitFloatVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitFloatVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<mediapipe::Image, false>
|
||||
SplitImageVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitImageVectorCalculator);
|
||||
|
||||
typedef SplitVectorCalculator<std::array<float, 16>, false>
|
||||
SplitAffineMatrixVectorCalculator;
|
||||
REGISTER_CALCULATOR(SplitAffineMatrixVectorCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -317,6 +317,7 @@ cc_library(
|
||||
cc_test(
|
||||
name = "image_cropping_calculator_test",
|
||||
srcs = ["image_cropping_calculator_test.cc"],
|
||||
tags = ["not_run:arm"],
|
||||
deps = [
|
||||
":image_cropping_calculator",
|
||||
":image_cropping_calculator_cc_proto",
|
||||
@@ -650,6 +651,7 @@ cc_library(
|
||||
cc_test(
|
||||
name = "segmentation_smoothing_calculator_test",
|
||||
srcs = ["segmentation_smoothing_calculator_test.cc"],
|
||||
tags = ["not_run:arm"],
|
||||
deps = [
|
||||
":image_clone_calculator",
|
||||
":image_clone_calculator_cc_proto",
|
||||
@@ -771,7 +773,10 @@ cc_test(
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero_interp_cubic.png",
|
||||
"//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
|
||||
],
|
||||
tags = ["desktop_only_test"],
|
||||
tags = [
|
||||
"desktop_only_test",
|
||||
"not_run:arm",
|
||||
],
|
||||
deps = [
|
||||
":affine_transformation",
|
||||
":image_transformation_calculator",
|
||||
|
||||
@@ -394,7 +394,7 @@ mediapipe_proto_library(
|
||||
# If you want to have precise control of which implementations to include (e.g. for strict binary
|
||||
# size concerns), depend on those implementations directly, and do not depend on
|
||||
# :inference_calculator.
|
||||
# In all cases, use "InferenceCalulator" in your graphs.
|
||||
# In all cases, use "InferenceCalculator" in your graphs.
|
||||
cc_library_with_tflite(
|
||||
name = "inference_calculator_interface",
|
||||
srcs = ["inference_calculator.cc"],
|
||||
@@ -655,6 +655,11 @@ cc_library(
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
|
||||
}) + select({
|
||||
"//mediapipe:apple": [
|
||||
"//third_party/apple_frameworks:MetalKit",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -1052,6 +1057,7 @@ cc_test(
|
||||
"testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
|
||||
"testdata/image_to_tensor/noop_except_range.png",
|
||||
],
|
||||
tags = ["not_run:arm"],
|
||||
deps = [
|
||||
":image_to_tensor_calculator",
|
||||
":image_to_tensor_converter",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -95,7 +95,8 @@ absl::Status FrameBufferProcessor::Convert(const mediapipe::Image& input,
|
||||
static_cast<int>(range_max) == 255);
|
||||
}
|
||||
|
||||
auto input_frame = input.GetGpuBuffer().GetReadView<FrameBuffer>();
|
||||
auto input_frame =
|
||||
input.GetGpuBuffer(/*upload_to_gpu=*/false).GetReadView<FrameBuffer>();
|
||||
const auto& output_shape = output_tensor.shape();
|
||||
MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
|
||||
FrameBuffer::Dimension output_dimension{/*width=*/output_shape.dims[2],
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -400,6 +400,16 @@ cc_library(
|
||||
# compile your binary with the flag TENSORFLOW_PROTOS=lite.
|
||||
cc_library(
|
||||
name = "tensorflow_inference_calculator_no_envelope_loader",
|
||||
deps = [
|
||||
":tensorflow_inference_calculator_for_boq",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
# This dependency removed tensorflow_jellyfish_deps and xprofilez_with_server because they failed
|
||||
# Boq conformance test. Weigh your use case to see if this will work for you.
|
||||
cc_library(
|
||||
name = "tensorflow_inference_calculator_for_boq",
|
||||
srcs = ["tensorflow_inference_calculator.cc"],
|
||||
deps = [
|
||||
":tensorflow_inference_calculator_cc_proto",
|
||||
@@ -585,6 +595,24 @@ cc_library(
|
||||
# See yaqs/1092546221614039040
|
||||
cc_library(
|
||||
name = "tensorflow_session_from_saved_model_generator_no_envelope_loader",
|
||||
defines = select({
|
||||
"//mediapipe:android": ["__ANDROID__"],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
deps = [
|
||||
":tensorflow_session_from_saved_model_generator_for_boq",
|
||||
] + select({
|
||||
"//conditions:default": [
|
||||
"//learning/brain/frameworks/uptc/public:uptc_session_no_envelope_loader",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
# Same library as tensorflow_session_from_saved_model_generator without uptc_session,
|
||||
# envelop_loader and remote_session dependencies.
|
||||
cc_library(
|
||||
name = "tensorflow_session_from_saved_model_generator_for_boq",
|
||||
srcs = ["tensorflow_session_from_saved_model_generator.cc"],
|
||||
defines = select({
|
||||
"//mediapipe:android": ["__ANDROID__"],
|
||||
|
||||
@@ -61,12 +61,12 @@ constexpr char kSessionBundleTag[] = "SESSION_BUNDLE";
|
||||
// overload GPU/TPU/...
|
||||
class SimpleSemaphore {
|
||||
public:
|
||||
explicit SimpleSemaphore(uint32 initial_count) : count_(initial_count) {}
|
||||
explicit SimpleSemaphore(uint32_t initial_count) : count_(initial_count) {}
|
||||
SimpleSemaphore(const SimpleSemaphore&) = delete;
|
||||
SimpleSemaphore(SimpleSemaphore&&) = delete;
|
||||
|
||||
// Acquires the semaphore by certain amount.
|
||||
void Acquire(uint32 amount) {
|
||||
void Acquire(uint32_t amount) {
|
||||
mutex_.Lock();
|
||||
while (count_ < amount) {
|
||||
cond_.Wait(&mutex_);
|
||||
@@ -76,7 +76,7 @@ class SimpleSemaphore {
|
||||
}
|
||||
|
||||
// Releases the semaphore by certain amount.
|
||||
void Release(uint32 amount) {
|
||||
void Release(uint32_t amount) {
|
||||
mutex_.Lock();
|
||||
count_ += amount;
|
||||
cond_.SignalAll();
|
||||
@@ -84,7 +84,7 @@ class SimpleSemaphore {
|
||||
}
|
||||
|
||||
private:
|
||||
uint32 count_;
|
||||
uint32_t count_;
|
||||
absl::Mutex mutex_;
|
||||
absl::CondVar cond_;
|
||||
};
|
||||
@@ -488,7 +488,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
// necessary.
|
||||
absl::Status OutputBatch(CalculatorContext* cc,
|
||||
std::unique_ptr<InferenceState> inference_state) {
|
||||
const int64 start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
const int64_t start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
std::vector<std::pair<mediapipe::ProtoString, tf::Tensor>> input_tensors;
|
||||
|
||||
for (auto& keyed_tensors : inference_state->input_tensor_batches_) {
|
||||
@@ -544,7 +544,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
get_session_run_throttle(options_.max_concurrent_session_runs());
|
||||
session_run_throttle->Acquire(1);
|
||||
}
|
||||
const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
const int64_t run_start_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
tf::Status tf_status;
|
||||
{
|
||||
#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
|
||||
@@ -562,7 +562,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
// informative error message.
|
||||
RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
|
||||
|
||||
const int64 run_end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
const int64_t run_end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
cc->GetCounter(kTotalSessionRunsTimeUsecsCounterSuffix)
|
||||
->IncrementBy(run_end_time - run_start_time);
|
||||
cc->GetCounter(kTotalNumSessionRunsCounterSuffix)->Increment();
|
||||
@@ -611,7 +611,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
}
|
||||
|
||||
// Get end time and report.
|
||||
const int64 end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
const int64_t end_time = absl::ToUnixMicros(clock_->TimeNow());
|
||||
cc->GetCounter(kTotalUsecsCounterSuffix)
|
||||
->IncrementBy(end_time - start_time);
|
||||
cc->GetCounter(kTotalProcessedTimestampsCounterSuffix)
|
||||
@@ -650,7 +650,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
|
||||
|
||||
// The static singleton semaphore to throttle concurrent session runs.
|
||||
static SimpleSemaphore* get_session_run_throttle(
|
||||
int32 max_concurrent_session_runs) {
|
||||
int32_t max_concurrent_session_runs) {
|
||||
static SimpleSemaphore* session_run_throttle =
|
||||
new SimpleSemaphore(max_concurrent_session_runs);
|
||||
return session_run_throttle;
|
||||
|
||||
@@ -197,15 +197,15 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
|
||||
// timestamp and the associated feature. This information is used in process
|
||||
// to output batches of packets in order.
|
||||
timestamps_.clear();
|
||||
int64 last_timestamp_seen = Timestamp::PreStream().Value();
|
||||
int64_t last_timestamp_seen = Timestamp::PreStream().Value();
|
||||
first_timestamp_seen_ = Timestamp::OneOverPostStream().Value();
|
||||
for (const auto& map_kv : sequence_->feature_lists().feature_list()) {
|
||||
if (absl::StrContains(map_kv.first, "/timestamp")) {
|
||||
LOG(INFO) << "Found feature timestamps: " << map_kv.first
|
||||
<< " with size: " << map_kv.second.feature_size();
|
||||
int64 recent_timestamp = Timestamp::PreStream().Value();
|
||||
int64_t recent_timestamp = Timestamp::PreStream().Value();
|
||||
for (int i = 0; i < map_kv.second.feature_size(); ++i) {
|
||||
int64 next_timestamp =
|
||||
int64_t next_timestamp =
|
||||
mpms::GetInt64sAt(*sequence_, map_kv.first, i).Get(0);
|
||||
RET_CHECK_GT(next_timestamp, recent_timestamp)
|
||||
<< "Timestamps must be sequential. If you're seeing this message "
|
||||
@@ -361,8 +361,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
|
||||
// any particular call to Process(). At the every end, we output the
|
||||
// poststream packets. If we only have poststream packets,
|
||||
// last_timestamp_key_ will be empty.
|
||||
int64 start_timestamp = 0;
|
||||
int64 end_timestamp = 0;
|
||||
int64_t start_timestamp = 0;
|
||||
int64_t end_timestamp = 0;
|
||||
if (last_timestamp_key_.empty() || process_poststream_) {
|
||||
process_poststream_ = true;
|
||||
start_timestamp = Timestamp::PostStream().Value();
|
||||
@@ -481,14 +481,14 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
|
||||
// Store a map from the keys for each stream to the timestamps for each
|
||||
// key. This allows us to identify which packets to output for each stream
|
||||
// for timestamps within a given time window.
|
||||
std::map<std::string, std::vector<int64>> timestamps_;
|
||||
std::map<std::string, std::vector<int64_t>> timestamps_;
|
||||
// Store the stream with the latest timestamp in the SequenceExample.
|
||||
std::string last_timestamp_key_;
|
||||
// Store the index of the current timestamp. Will be less than
|
||||
// timestamps_[last_timestamp_key_].size().
|
||||
int current_timestamp_index_;
|
||||
// Store the very first timestamp, so we output everything on the first frame.
|
||||
int64 first_timestamp_seen_;
|
||||
int64_t first_timestamp_seen_;
|
||||
// List of keypoint names.
|
||||
std::vector<std::string> keypoint_names_;
|
||||
// Default keypoint location when missing.
|
||||
|
||||
@@ -54,7 +54,7 @@ class VectorToTensorFloatCalculatorTest : public ::testing::Test {
|
||||
}
|
||||
}
|
||||
|
||||
const int64 time = 1234;
|
||||
const int64_t time = 1234;
|
||||
runner_->MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(input.release()).At(Timestamp(time)));
|
||||
|
||||
@@ -91,7 +91,7 @@ TEST_F(VectorToTensorFloatCalculatorTest, ConvertsFromVectorFloat) {
|
||||
// 2^i can be represented exactly in floating point numbers if 'i' is small.
|
||||
input->at(i) = static_cast<float>(1 << i);
|
||||
}
|
||||
const int64 time = 1234;
|
||||
const int64_t time = 1234;
|
||||
runner_->MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(input.release()).At(Timestamp(time)));
|
||||
|
||||
|
||||
@@ -899,16 +899,77 @@ mediapipe_proto_library(
|
||||
cc_library(
|
||||
name = "landmarks_smoothing_calculator",
|
||||
srcs = ["landmarks_smoothing_calculator.cc"],
|
||||
hdrs = ["landmarks_smoothing_calculator.h"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_cc_proto",
|
||||
":landmarks_smoothing_calculator_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "landmarks_smoothing_calculator_utils",
|
||||
srcs = ["landmarks_smoothing_calculator_utils.cc"],
|
||||
hdrs = ["landmarks_smoothing_calculator_utils.h"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util/filtering:one_euro_filter",
|
||||
"//mediapipe/util/filtering:relative_velocity_filter",
|
||||
"@com_google_absl//absl/algorithm:container",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "landmarks_smoothing_calculator_utils_test",
|
||||
size = "small",
|
||||
srcs = ["landmarks_smoothing_calculator_utils_test.cc"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_utils",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "multi_landmarks_smoothing_calculator",
|
||||
srcs = ["multi_landmarks_smoothing_calculator.cc"],
|
||||
hdrs = ["multi_landmarks_smoothing_calculator.h"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_cc_proto",
|
||||
":landmarks_smoothing_calculator_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "multi_world_landmarks_smoothing_calculator",
|
||||
srcs = ["multi_world_landmarks_smoothing_calculator.cc"],
|
||||
hdrs = ["multi_world_landmarks_smoothing_calculator.h"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_cc_proto",
|
||||
":landmarks_smoothing_calculator_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -1285,12 +1346,14 @@ cc_library(
|
||||
srcs = ["flat_color_image_calculator.cc"],
|
||||
deps = [
|
||||
":flat_color_image_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/api2:node",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/util:color_cc_proto",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/strings",
|
||||
|
||||
@@ -471,7 +471,7 @@ absl::Status AnnotationOverlayCalculator::CreateRenderTargetCpu(
|
||||
auto input_mat = formats::MatView(&input_frame);
|
||||
if (input_frame.Format() == ImageFormat::GRAY8) {
|
||||
cv::Mat rgb_mat;
|
||||
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
|
||||
cv::cvtColor(input_mat, rgb_mat, cv::COLOR_GRAY2RGB);
|
||||
rgb_mat.copyTo(*image_mat);
|
||||
} else {
|
||||
input_mat.copyTo(*image_mat);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -15,14 +15,13 @@
|
||||
#include <memory>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/util/flat_color_image_calculator.pb.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
#include "mediapipe/util/color.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
@@ -32,6 +31,7 @@ namespace {
|
||||
using ::mediapipe::api2::Input;
|
||||
using ::mediapipe::api2::Node;
|
||||
using ::mediapipe::api2::Output;
|
||||
using ::mediapipe::api2::SideOutput;
|
||||
} // namespace
|
||||
|
||||
// A calculator for generating an image filled with a single color.
|
||||
@@ -45,7 +45,8 @@ using ::mediapipe::api2::Output;
|
||||
//
|
||||
// Outputs:
|
||||
// IMAGE (Image)
|
||||
// Image filled with the requested color.
|
||||
// Image filled with the requested color. Can be either an output_stream
|
||||
// or an output_side_packet.
|
||||
//
|
||||
// Example useage:
|
||||
// node {
|
||||
@@ -68,9 +69,10 @@ class FlatColorImageCalculator : public Node {
|
||||
public:
|
||||
static constexpr Input<Image>::Optional kInImage{"IMAGE"};
|
||||
static constexpr Input<Color>::Optional kInColor{"COLOR"};
|
||||
static constexpr Output<Image> kOutImage{"IMAGE"};
|
||||
static constexpr Output<Image>::Optional kOutImage{"IMAGE"};
|
||||
static constexpr SideOutput<Image>::Optional kOutSideImage{"IMAGE"};
|
||||
|
||||
MEDIAPIPE_NODE_CONTRACT(kInImage, kInColor, kOutImage);
|
||||
MEDIAPIPE_NODE_CONTRACT(kInImage, kInColor, kOutImage, kOutSideImage);
|
||||
|
||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||
const auto& options = cc->Options<FlatColorImageCalculatorOptions>();
|
||||
@@ -81,6 +83,13 @@ class FlatColorImageCalculator : public Node {
|
||||
RET_CHECK(kInColor(cc).IsConnected() ^ options.has_color())
|
||||
<< "Either set COLOR input stream, or set through options";
|
||||
|
||||
RET_CHECK(kOutImage(cc).IsConnected() ^ kOutSideImage(cc).IsConnected())
|
||||
<< "Set IMAGE either as output stream, or as output side packet";
|
||||
|
||||
RET_CHECK(!kOutSideImage(cc).IsConnected() ||
|
||||
(options.has_output_height() && options.has_output_width()))
|
||||
<< "Set size through options, when setting IMAGE as output side packet";
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
@@ -88,6 +97,9 @@ class FlatColorImageCalculator : public Node {
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
std::optional<std::shared_ptr<ImageFrame>> CreateOutputFrame(
|
||||
CalculatorContext* cc);
|
||||
|
||||
bool use_dimension_from_option_ = false;
|
||||
bool use_color_from_option_ = false;
|
||||
};
|
||||
@@ -96,10 +108,31 @@ MEDIAPIPE_REGISTER_NODE(FlatColorImageCalculator);
|
||||
absl::Status FlatColorImageCalculator::Open(CalculatorContext* cc) {
|
||||
use_dimension_from_option_ = !kInImage(cc).IsConnected();
|
||||
use_color_from_option_ = !kInColor(cc).IsConnected();
|
||||
|
||||
if (!kOutImage(cc).IsConnected()) {
|
||||
std::optional<std::shared_ptr<ImageFrame>> output_frame =
|
||||
CreateOutputFrame(cc);
|
||||
if (output_frame.has_value()) {
|
||||
kOutSideImage(cc).Set(Image(output_frame.value()));
|
||||
}
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status FlatColorImageCalculator::Process(CalculatorContext* cc) {
|
||||
if (kOutImage(cc).IsConnected()) {
|
||||
std::optional<std::shared_ptr<ImageFrame>> output_frame =
|
||||
CreateOutputFrame(cc);
|
||||
if (output_frame.has_value()) {
|
||||
kOutImage(cc).Send(Image(output_frame.value()));
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
std::optional<std::shared_ptr<ImageFrame>>
|
||||
FlatColorImageCalculator::CreateOutputFrame(CalculatorContext* cc) {
|
||||
const auto& options = cc->Options<FlatColorImageCalculatorOptions>();
|
||||
|
||||
int output_height = -1;
|
||||
@@ -112,7 +145,7 @@ absl::Status FlatColorImageCalculator::Process(CalculatorContext* cc) {
|
||||
output_height = input_image.height();
|
||||
output_width = input_image.width();
|
||||
} else {
|
||||
return absl::OkStatus();
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
Color color;
|
||||
@@ -121,7 +154,7 @@ absl::Status FlatColorImageCalculator::Process(CalculatorContext* cc) {
|
||||
} else if (!kInColor(cc).IsEmpty()) {
|
||||
color = kInColor(cc).Get();
|
||||
} else {
|
||||
return absl::OkStatus();
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto output_frame = std::make_shared<ImageFrame>(ImageFormat::SRGB,
|
||||
@@ -130,9 +163,7 @@ absl::Status FlatColorImageCalculator::Process(CalculatorContext* cc) {
|
||||
|
||||
output_mat.setTo(cv::Scalar(color.r(), color.g(), color.b()));
|
||||
|
||||
kOutImage(cc).Send(Image(output_frame));
|
||||
|
||||
return absl::OkStatus();
|
||||
return output_frame;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -113,6 +113,35 @@ TEST(FlatColorImageCalculatorTest, SpecifyDimensionThroughOptions) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(FlatColorImageCalculatorTest, ProducesOutputSidePacket) {
|
||||
CalculatorRunner runner(R"pb(
|
||||
calculator: "FlatColorImageCalculator"
|
||||
output_side_packet: "IMAGE:out_packet"
|
||||
options {
|
||||
[mediapipe.FlatColorImageCalculatorOptions.ext] {
|
||||
output_width: 1
|
||||
output_height: 1
|
||||
color: {
|
||||
r: 100,
|
||||
g: 200,
|
||||
b: 255,
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
|
||||
const auto& image = runner.OutputSidePackets().Tag(kImageTag).Get<Image>();
|
||||
EXPECT_EQ(image.width(), 1);
|
||||
EXPECT_EQ(image.height(), 1);
|
||||
auto image_frame = image.GetImageFrameSharedPtr();
|
||||
const uint8_t* pixel_data = image_frame->PixelData();
|
||||
EXPECT_EQ(pixel_data[0], 100);
|
||||
EXPECT_EQ(pixel_data[1], 200);
|
||||
EXPECT_EQ(pixel_data[2], 255);
|
||||
}
|
||||
|
||||
TEST(FlatColorImageCalculatorTest, FailureMissingDimension) {
|
||||
CalculatorRunner runner(R"pb(
|
||||
calculator: "FlatColorImageCalculator"
|
||||
@@ -206,5 +235,56 @@ TEST(FlatColorImageCalculatorTest, FailureDuplicateColor) {
|
||||
HasSubstr("Either set COLOR input stream"));
|
||||
}
|
||||
|
||||
TEST(FlatColorImageCalculatorTest, FailureDuplicateOutputs) {
|
||||
CalculatorRunner runner(R"pb(
|
||||
calculator: "FlatColorImageCalculator"
|
||||
output_stream: "IMAGE:out_image"
|
||||
output_side_packet: "IMAGE:out_packet"
|
||||
options {
|
||||
[mediapipe.FlatColorImageCalculatorOptions.ext] {
|
||||
output_width: 1
|
||||
output_height: 1
|
||||
color: {
|
||||
r: 100,
|
||||
g: 200,
|
||||
b: 255,
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
|
||||
ASSERT_THAT(
|
||||
runner.Run().message(),
|
||||
HasSubstr("Set IMAGE either as output stream, or as output side packet"));
|
||||
}
|
||||
|
||||
TEST(FlatColorImageCalculatorTest, FailureSettingInputImageOnOutputSidePacket) {
|
||||
CalculatorRunner runner(R"pb(
|
||||
calculator: "FlatColorImageCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
output_side_packet: "IMAGE:out_packet"
|
||||
options {
|
||||
[mediapipe.FlatColorImageCalculatorOptions.ext] {
|
||||
color: {
|
||||
r: 100,
|
||||
g: 200,
|
||||
b: 255,
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
|
||||
auto image_frame = std::make_shared<ImageFrame>(ImageFormat::SRGB,
|
||||
kImageWidth, kImageHeight);
|
||||
|
||||
for (int ts = 0; ts < 3; ++ts) {
|
||||
runner.MutableInputs()->Tag(kImageTag).packets.push_back(
|
||||
MakePacket<Image>(image_frame).At(Timestamp(ts)));
|
||||
}
|
||||
ASSERT_THAT(runner.Run().message(),
|
||||
HasSubstr("Set size through options, when setting IMAGE as "
|
||||
"output side packet"));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -12,471 +12,105 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/util/filtering/one_euro_filter.h"
|
||||
#include "mediapipe/util/filtering/relative_velocity_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
|
||||
constexpr char kLandmarksTag[] = "LANDMARKS";
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
constexpr char kObjectScaleRoiTag[] = "OBJECT_SCALE_ROI";
|
||||
constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
|
||||
constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
|
||||
|
||||
using ::mediapipe::NormalizedRect;
|
||||
using mediapipe::OneEuroFilter;
|
||||
using ::mediapipe::Rect;
|
||||
using mediapipe::RelativeVelocityFilter;
|
||||
|
||||
void NormalizedLandmarksToLandmarks(
|
||||
const NormalizedLandmarkList& norm_landmarks, const int image_width,
|
||||
const int image_height, LandmarkList* landmarks) {
|
||||
for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
|
||||
const auto& norm_landmark = norm_landmarks.landmark(i);
|
||||
|
||||
auto* landmark = landmarks->add_landmark();
|
||||
landmark->set_x(norm_landmark.x() * image_width);
|
||||
landmark->set_y(norm_landmark.y() * image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
landmark->set_z(norm_landmark.z() * image_width);
|
||||
landmark->set_visibility(norm_landmark.visibility());
|
||||
landmark->set_presence(norm_landmark.presence());
|
||||
}
|
||||
}
|
||||
|
||||
void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
|
||||
const int image_width,
|
||||
const int image_height,
|
||||
NormalizedLandmarkList* norm_landmarks) {
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const auto& landmark = landmarks.landmark(i);
|
||||
|
||||
auto* norm_landmark = norm_landmarks->add_landmark();
|
||||
norm_landmark->set_x(landmark.x() / image_width);
|
||||
norm_landmark->set_y(landmark.y() / image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
norm_landmark->set_z(landmark.z() / image_width);
|
||||
norm_landmark->set_visibility(landmark.visibility());
|
||||
norm_landmark->set_presence(landmark.presence());
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate object scale to use its inverse value as velocity scale for
|
||||
// RelativeVelocityFilter. If value will be too small (less than
|
||||
// `options_.min_allowed_object_scale`) smoothing will be disabled and
|
||||
// landmarks will be returned as is.
|
||||
// Object scale is calculated as average between bounding box width and height
|
||||
// with sides parallel to axis.
|
||||
float GetObjectScale(const LandmarkList& landmarks) {
|
||||
const auto& lm_minmax_x = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.x() < b.x(); });
|
||||
const float x_min = lm_minmax_x.first->x();
|
||||
const float x_max = lm_minmax_x.second->x();
|
||||
|
||||
const auto& lm_minmax_y = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.y() < b.y(); });
|
||||
const float y_min = lm_minmax_y.first->y();
|
||||
const float y_max = lm_minmax_y.second->y();
|
||||
|
||||
const float object_width = x_max - x_min;
|
||||
const float object_height = y_max - y_min;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
|
||||
float GetObjectScale(const NormalizedRect& roi, const int image_width,
|
||||
const int image_height) {
|
||||
const float object_width = roi.width() * image_width;
|
||||
const float object_height = roi.height() * image_height;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
|
||||
float GetObjectScale(const Rect& roi) {
|
||||
return (roi.width() + roi.height()) / 2.0f;
|
||||
}
|
||||
|
||||
// Abstract class for various landmarks filters.
|
||||
class LandmarksFilter {
|
||||
public:
|
||||
virtual ~LandmarksFilter() = default;
|
||||
|
||||
virtual absl::Status Reset() { return absl::OkStatus(); }
|
||||
|
||||
virtual absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList* out_landmarks) = 0;
|
||||
};
|
||||
|
||||
// Returns landmarks as is without smoothing.
|
||||
class NoFilter : public LandmarksFilter {
|
||||
public:
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList* out_landmarks) override {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
// Please check RelativeVelocityFilter documentation for details.
|
||||
class VelocityFilter : public LandmarksFilter {
|
||||
public:
|
||||
VelocityFilter(int window_size, float velocity_scale,
|
||||
float min_allowed_object_scale, bool disable_value_scaling)
|
||||
: window_size_(window_size),
|
||||
velocity_scale_(velocity_scale),
|
||||
min_allowed_object_scale_(min_allowed_object_scale),
|
||||
disable_value_scaling_(disable_value_scaling) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList* out_landmarks) override {
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
float value_scale = 1.0f;
|
||||
if (!disable_value_scaling_) {
|
||||
const float object_scale =
|
||||
object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
value_scale = 1.0f / object_scale;
|
||||
}
|
||||
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks->add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(
|
||||
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
|
||||
out_landmark->set_y(
|
||||
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
|
||||
out_landmark->set_z(
|
||||
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
x_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
y_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
z_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
int window_size_;
|
||||
float velocity_scale_;
|
||||
float min_allowed_object_scale_;
|
||||
bool disable_value_scaling_;
|
||||
|
||||
std::vector<RelativeVelocityFilter> x_filters_;
|
||||
std::vector<RelativeVelocityFilter> y_filters_;
|
||||
std::vector<RelativeVelocityFilter> z_filters_;
|
||||
};
|
||||
|
||||
// Please check OneEuroFilter documentation for details.
|
||||
class OneEuroFilterImpl : public LandmarksFilter {
|
||||
public:
|
||||
OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
|
||||
double derivate_cutoff, float min_allowed_object_scale,
|
||||
bool disable_value_scaling)
|
||||
: frequency_(frequency),
|
||||
min_cutoff_(min_cutoff),
|
||||
beta_(beta),
|
||||
derivate_cutoff_(derivate_cutoff),
|
||||
min_allowed_object_scale_(min_allowed_object_scale),
|
||||
disable_value_scaling_(disable_value_scaling) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList* out_landmarks) override {
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
float value_scale = 1.0f;
|
||||
if (!disable_value_scaling_) {
|
||||
const float object_scale =
|
||||
object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
*out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
value_scale = 1.0f / object_scale;
|
||||
}
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks->add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(
|
||||
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
|
||||
out_landmark->set_y(
|
||||
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
|
||||
out_landmark->set_z(
|
||||
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_landmarks; ++i) {
|
||||
x_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
y_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
z_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
double frequency_;
|
||||
double min_cutoff_;
|
||||
double beta_;
|
||||
double derivate_cutoff_;
|
||||
double min_allowed_object_scale_;
|
||||
bool disable_value_scaling_;
|
||||
|
||||
std::vector<OneEuroFilter> x_filters_;
|
||||
std::vector<OneEuroFilter> y_filters_;
|
||||
std::vector<OneEuroFilter> z_filters_;
|
||||
};
|
||||
using ::mediapipe::landmarks_smoothing::GetObjectScale;
|
||||
using ::mediapipe::landmarks_smoothing::InitializeLandmarksFilter;
|
||||
using ::mediapipe::landmarks_smoothing::LandmarksFilter;
|
||||
using ::mediapipe::landmarks_smoothing::LandmarksToNormalizedLandmarks;
|
||||
using ::mediapipe::landmarks_smoothing::NormalizedLandmarksToLandmarks;
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
//
|
||||
// Inputs:
|
||||
// NORM_LANDMARKS: A NormalizedLandmarkList of landmarks you want to smooth.
|
||||
// IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
|
||||
// Required to perform all computations in absolute coordinates to avoid any
|
||||
// influence of normalized values.
|
||||
// OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
|
||||
// input landmarks) used to determine the object scale for some of the
|
||||
// filters. If not provided - object scale will be calculated from
|
||||
// landmarks.
|
||||
//
|
||||
// Outputs:
|
||||
// NORM_FILTERED_LANDMARKS: A NormalizedLandmarkList of smoothed landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "LandmarksSmoothingCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:pose_landmarks"
|
||||
// input_stream: "IMAGE_SIZE:image_size"
|
||||
// input_stream: "OBJECT_SCALE_ROI:roi"
|
||||
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
|
||||
// options: {
|
||||
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
// velocity_filter: {
|
||||
// window_size: 5
|
||||
// velocity_scale: 10.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class LandmarksSmoothingCalculator : public CalculatorBase {
|
||||
class LandmarksSmoothingCalculatorImpl
|
||||
: public NodeImpl<LandmarksSmoothingCalculator> {
|
||||
public:
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
absl::Status Open(CalculatorContext* cc) override;
|
||||
absl::Status Process(CalculatorContext* cc) override;
|
||||
absl::Status Open(CalculatorContext* cc) override {
|
||||
ASSIGN_OR_RETURN(landmarks_filter_,
|
||||
InitializeLandmarksFilter(
|
||||
cc->Options<LandmarksSmoothingCalculatorOptions>()));
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if ((kInNormLandmarks(cc).IsConnected() &&
|
||||
kInNormLandmarks(cc).IsEmpty()) ||
|
||||
(kInLandmarks(cc).IsConnected() && kInLandmarks(cc).IsEmpty())) {
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
if (kInNormLandmarks(cc).IsConnected()) {
|
||||
const auto& in_norm_landmarks = kInNormLandmarks(cc).Get();
|
||||
|
||||
int image_width;
|
||||
int image_height;
|
||||
std::tie(image_width, image_height) = kImageSize(cc).Get();
|
||||
|
||||
absl::optional<float> object_scale;
|
||||
if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
|
||||
auto& roi = kObjectScaleRoi(cc).Get<NormalizedRect>();
|
||||
object_scale = GetObjectScale(roi, image_width, image_height);
|
||||
}
|
||||
|
||||
auto in_landmarks = absl::make_unique<LandmarkList>();
|
||||
NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width,
|
||||
image_height, *in_landmarks.get());
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
|
||||
*in_landmarks, timestamp, object_scale, *out_landmarks));
|
||||
|
||||
auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
|
||||
LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
|
||||
*out_norm_landmarks.get());
|
||||
|
||||
kOutNormLandmarks(cc).Send(std::move(out_norm_landmarks));
|
||||
} else {
|
||||
const auto& in_landmarks = kInLandmarks(cc).Get();
|
||||
|
||||
absl::optional<float> object_scale;
|
||||
if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
|
||||
auto& roi = kObjectScaleRoi(cc).Get<Rect>();
|
||||
object_scale = GetObjectScale(roi);
|
||||
}
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
|
||||
in_landmarks, timestamp, object_scale, *out_landmarks));
|
||||
|
||||
kOutLandmarks(cc).Send(std::move(out_landmarks));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<LandmarksFilter> landmarks_filter_;
|
||||
};
|
||||
REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
|
||||
|
||||
absl::Status LandmarksSmoothingCalculator::GetContract(CalculatorContract* cc) {
|
||||
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Set<NormalizedLandmarkList>();
|
||||
|
||||
if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
|
||||
cc->Inputs().Tag(kObjectScaleRoiTag).Set<NormalizedRect>();
|
||||
}
|
||||
} else {
|
||||
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
|
||||
cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
|
||||
|
||||
if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
|
||||
cc->Inputs().Tag(kObjectScaleRoiTag).Set<Rect>();
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
// Pick landmarks filter.
|
||||
const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
|
||||
if (options.has_no_filter()) {
|
||||
landmarks_filter_ = absl::make_unique<NoFilter>();
|
||||
} else if (options.has_velocity_filter()) {
|
||||
landmarks_filter_ = absl::make_unique<VelocityFilter>(
|
||||
options.velocity_filter().window_size(),
|
||||
options.velocity_filter().velocity_scale(),
|
||||
options.velocity_filter().min_allowed_object_scale(),
|
||||
options.velocity_filter().disable_value_scaling());
|
||||
} else if (options.has_one_euro_filter()) {
|
||||
landmarks_filter_ = absl::make_unique<OneEuroFilterImpl>(
|
||||
options.one_euro_filter().frequency(),
|
||||
options.one_euro_filter().min_cutoff(),
|
||||
options.one_euro_filter().beta(),
|
||||
options.one_euro_filter().derivate_cutoff(),
|
||||
options.one_euro_filter().min_allowed_object_scale(),
|
||||
options.one_euro_filter().disable_value_scaling());
|
||||
} else {
|
||||
RET_CHECK_FAIL()
|
||||
<< "Landmarks filter is either not specified or not supported";
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
|
||||
(cc->Inputs().HasTag(kLandmarksTag) &&
|
||||
cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
|
||||
const auto& in_norm_landmarks =
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
|
||||
int image_width;
|
||||
int image_height;
|
||||
std::tie(image_width, image_height) =
|
||||
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
|
||||
absl::optional<float> object_scale;
|
||||
if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
|
||||
!cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
|
||||
auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<NormalizedRect>();
|
||||
object_scale = GetObjectScale(roi, image_width, image_height);
|
||||
}
|
||||
|
||||
auto in_landmarks = absl::make_unique<LandmarkList>();
|
||||
NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width, image_height,
|
||||
in_landmarks.get());
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
|
||||
*in_landmarks, timestamp, object_scale, out_landmarks.get()));
|
||||
|
||||
auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
|
||||
LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
|
||||
out_norm_landmarks.get());
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Add(out_norm_landmarks.release(), cc->InputTimestamp());
|
||||
} else {
|
||||
const auto& in_landmarks =
|
||||
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
|
||||
|
||||
absl::optional<float> object_scale;
|
||||
if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
|
||||
!cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
|
||||
auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<Rect>();
|
||||
object_scale = GetObjectScale(roi);
|
||||
}
|
||||
|
||||
auto out_landmarks = absl::make_unique<LandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
|
||||
in_landmarks, timestamp, object_scale, out_landmarks.get()));
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kFilteredLandmarksTag)
|
||||
.Add(out_landmarks.release(), cc->InputTimestamp());
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
MEDIAPIPE_NODE_IMPLEMENTATION(LandmarksSmoothingCalculatorImpl);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
//
|
||||
// Inputs:
|
||||
// NORM_LANDMARKS (optional): A NormalizedLandmarkList of landmarks you want
|
||||
// to smooth.
|
||||
// LANDMARKS (optional): A LandmarkList of landmarks you want to smooth.
|
||||
// IMAGE_SIZE (optional): A std::pair<int, int> represention of image width
|
||||
// and height. Required to perform all computations in absolute coordinates
|
||||
// when smoothing NORM_LANDMARKS to avoid any influence of normalized
|
||||
// values.
|
||||
// OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
|
||||
// input landmarks) used to determine the object scale for some of the
|
||||
// filters. If not provided - object scale will be calculated from
|
||||
// landmarks.
|
||||
//
|
||||
// Outputs:
|
||||
// NORM_FILTERED_LANDMARKS (optional): A NormalizedLandmarkList of smoothed
|
||||
// landmarks.
|
||||
// FILTERED_LANDMARKS (optional): A LandmarkList of smoothed landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "LandmarksSmoothingCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:landmarks"
|
||||
// input_stream: "IMAGE_SIZE:image_size"
|
||||
// input_stream: "OBJECT_SCALE_ROI:roi"
|
||||
// output_stream: "NORM_FILTERED_LANDMARKS:landmarks_filtered"
|
||||
// options: {
|
||||
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
// velocity_filter: {
|
||||
// window_size: 5
|
||||
// velocity_scale: 10.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class LandmarksSmoothingCalculator : public NodeIntf {
|
||||
public:
|
||||
static constexpr Input<mediapipe::NormalizedLandmarkList>::Optional
|
||||
kInNormLandmarks{"NORM_LANDMARKS"};
|
||||
static constexpr Input<mediapipe::LandmarkList>::Optional kInLandmarks{
|
||||
"LANDMARKS"};
|
||||
static constexpr Input<std::pair<int, int>>::Optional kImageSize{
|
||||
"IMAGE_SIZE"};
|
||||
static constexpr Input<OneOf<NormalizedRect, Rect>>::Optional kObjectScaleRoi{
|
||||
"OBJECT_SCALE_ROI"};
|
||||
static constexpr Output<mediapipe::NormalizedLandmarkList>::Optional
|
||||
kOutNormLandmarks{"NORM_FILTERED_LANDMARKS"};
|
||||
static constexpr Output<mediapipe::LandmarkList>::Optional kOutLandmarks{
|
||||
"FILTERED_LANDMARKS"};
|
||||
MEDIAPIPE_NODE_INTERFACE(LandmarksSmoothingCalculator, kInNormLandmarks,
|
||||
kInLandmarks, kImageSize, kObjectScaleRoi,
|
||||
kOutNormLandmarks, kOutLandmarks);
|
||||
|
||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||
RET_CHECK(kInNormLandmarks(cc).IsConnected() ^
|
||||
kInLandmarks(cc).IsConnected())
|
||||
<< "One and only one of NORM_LANDMARKS and LANDMARKS input is allowed";
|
||||
|
||||
// TODO: Verify scale ROI is of the same type as landmarks
|
||||
// that are being smoothed.
|
||||
|
||||
if (kInNormLandmarks(cc).IsConnected()) {
|
||||
RET_CHECK(kImageSize(cc).IsConnected());
|
||||
RET_CHECK(kOutNormLandmarks(cc).IsConnected());
|
||||
RET_CHECK(!kOutLandmarks(cc).IsConnected());
|
||||
} else {
|
||||
RET_CHECK(!kImageSize(cc).IsConnected());
|
||||
RET_CHECK(kOutLandmarks(cc).IsConnected());
|
||||
RET_CHECK(!kOutNormLandmarks(cc).IsConnected());
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
@@ -0,0 +1,375 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/util/filtering/one_euro_filter.h"
|
||||
#include "mediapipe/util/filtering/relative_velocity_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace landmarks_smoothing {
|
||||
|
||||
namespace {
|
||||
|
||||
using ::mediapipe::NormalizedRect;
|
||||
using ::mediapipe::OneEuroFilter;
|
||||
using ::mediapipe::Rect;
|
||||
using ::mediapipe::RelativeVelocityFilter;
|
||||
|
||||
// Estimate object scale to use its inverse value as velocity scale for
|
||||
// RelativeVelocityFilter. If value will be too small (less than
|
||||
// `options_.min_allowed_object_scale`) smoothing will be disabled and
|
||||
// landmarks will be returned as is.
|
||||
// Object scale is calculated as average between bounding box width and height
|
||||
// with sides parallel to axis.
|
||||
float GetObjectScale(const LandmarkList& landmarks) {
|
||||
const auto& lm_minmax_x = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.x() < b.x(); });
|
||||
const float x_min = lm_minmax_x.first->x();
|
||||
const float x_max = lm_minmax_x.second->x();
|
||||
|
||||
const auto& lm_minmax_y = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.y() < b.y(); });
|
||||
const float y_min = lm_minmax_y.first->y();
|
||||
const float y_max = lm_minmax_y.second->y();
|
||||
|
||||
const float object_width = x_max - x_min;
|
||||
const float object_height = y_max - y_min;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
|
||||
// Returns landmarks as is without smoothing.
|
||||
class NoFilter : public LandmarksFilter {
|
||||
public:
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList& out_landmarks) override {
|
||||
out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
// Please check RelativeVelocityFilter documentation for details.
|
||||
class VelocityFilter : public LandmarksFilter {
|
||||
public:
|
||||
VelocityFilter(int window_size, float velocity_scale,
|
||||
float min_allowed_object_scale, bool disable_value_scaling)
|
||||
: window_size_(window_size),
|
||||
velocity_scale_(velocity_scale),
|
||||
min_allowed_object_scale_(min_allowed_object_scale),
|
||||
disable_value_scaling_(disable_value_scaling) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList& out_landmarks) override {
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
float value_scale = 1.0f;
|
||||
if (!disable_value_scaling_) {
|
||||
const float object_scale =
|
||||
object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
value_scale = 1.0f / object_scale;
|
||||
}
|
||||
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks.add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(
|
||||
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
|
||||
out_landmark->set_y(
|
||||
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
|
||||
out_landmark->set_z(
|
||||
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
x_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
y_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
z_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
int window_size_;
|
||||
float velocity_scale_;
|
||||
float min_allowed_object_scale_;
|
||||
bool disable_value_scaling_;
|
||||
|
||||
std::vector<RelativeVelocityFilter> x_filters_;
|
||||
std::vector<RelativeVelocityFilter> y_filters_;
|
||||
std::vector<RelativeVelocityFilter> z_filters_;
|
||||
};
|
||||
|
||||
// Please check OneEuroFilter documentation for details.
|
||||
class OneEuroFilterImpl : public LandmarksFilter {
|
||||
public:
|
||||
OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
|
||||
double derivate_cutoff, float min_allowed_object_scale,
|
||||
bool disable_value_scaling)
|
||||
: frequency_(frequency),
|
||||
min_cutoff_(min_cutoff),
|
||||
beta_(beta),
|
||||
derivate_cutoff_(derivate_cutoff),
|
||||
min_allowed_object_scale_(min_allowed_object_scale),
|
||||
disable_value_scaling_(disable_value_scaling) {}
|
||||
|
||||
absl::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status Apply(const LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
LandmarkList& out_landmarks) override {
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
float value_scale = 1.0f;
|
||||
if (!disable_value_scaling_) {
|
||||
const float object_scale =
|
||||
object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
out_landmarks = in_landmarks;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
value_scale = 1.0f / object_scale;
|
||||
}
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const auto& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
auto* out_landmark = out_landmarks.add_landmark();
|
||||
*out_landmark = in_landmark;
|
||||
out_landmark->set_x(
|
||||
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
|
||||
out_landmark->set_y(
|
||||
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
|
||||
out_landmark->set_z(
|
||||
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_landmarks; ++i) {
|
||||
x_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
y_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
z_filters_.push_back(
|
||||
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
double frequency_;
|
||||
double min_cutoff_;
|
||||
double beta_;
|
||||
double derivate_cutoff_;
|
||||
double min_allowed_object_scale_;
|
||||
bool disable_value_scaling_;
|
||||
|
||||
std::vector<OneEuroFilter> x_filters_;
|
||||
std::vector<OneEuroFilter> y_filters_;
|
||||
std::vector<OneEuroFilter> z_filters_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void NormalizedLandmarksToLandmarks(
|
||||
const NormalizedLandmarkList& norm_landmarks, const int image_width,
|
||||
const int image_height, LandmarkList& landmarks) {
|
||||
for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
|
||||
const auto& norm_landmark = norm_landmarks.landmark(i);
|
||||
|
||||
auto* landmark = landmarks.add_landmark();
|
||||
landmark->set_x(norm_landmark.x() * image_width);
|
||||
landmark->set_y(norm_landmark.y() * image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
landmark->set_z(norm_landmark.z() * image_width);
|
||||
|
||||
if (norm_landmark.has_visibility()) {
|
||||
landmark->set_visibility(norm_landmark.visibility());
|
||||
} else {
|
||||
landmark->clear_visibility();
|
||||
}
|
||||
|
||||
if (norm_landmark.has_presence()) {
|
||||
landmark->set_presence(norm_landmark.presence());
|
||||
} else {
|
||||
landmark->clear_presence();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
|
||||
const int image_width,
|
||||
const int image_height,
|
||||
NormalizedLandmarkList& norm_landmarks) {
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const auto& landmark = landmarks.landmark(i);
|
||||
|
||||
auto* norm_landmark = norm_landmarks.add_landmark();
|
||||
norm_landmark->set_x(landmark.x() / image_width);
|
||||
norm_landmark->set_y(landmark.y() / image_height);
|
||||
// Scale Z the same way as X (using image width).
|
||||
norm_landmark->set_z(landmark.z() / image_width);
|
||||
|
||||
if (landmark.has_visibility()) {
|
||||
norm_landmark->set_visibility(landmark.visibility());
|
||||
} else {
|
||||
norm_landmark->clear_visibility();
|
||||
}
|
||||
|
||||
if (landmark.has_presence()) {
|
||||
norm_landmark->set_presence(landmark.presence());
|
||||
} else {
|
||||
norm_landmark->clear_presence();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
float GetObjectScale(const NormalizedRect& roi, const int image_width,
|
||||
const int image_height) {
|
||||
const float object_width = roi.width() * image_width;
|
||||
const float object_height = roi.height() * image_height;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
|
||||
float GetObjectScale(const Rect& roi) {
|
||||
return (roi.width() + roi.height()) / 2.0f;
|
||||
}
|
||||
|
||||
absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
|
||||
const LandmarksSmoothingCalculatorOptions& options) {
|
||||
if (options.has_no_filter()) {
|
||||
return absl::make_unique<NoFilter>();
|
||||
} else if (options.has_velocity_filter()) {
|
||||
return absl::make_unique<VelocityFilter>(
|
||||
options.velocity_filter().window_size(),
|
||||
options.velocity_filter().velocity_scale(),
|
||||
options.velocity_filter().min_allowed_object_scale(),
|
||||
options.velocity_filter().disable_value_scaling());
|
||||
} else if (options.has_one_euro_filter()) {
|
||||
return absl::make_unique<OneEuroFilterImpl>(
|
||||
options.one_euro_filter().frequency(),
|
||||
options.one_euro_filter().min_cutoff(),
|
||||
options.one_euro_filter().beta(),
|
||||
options.one_euro_filter().derivate_cutoff(),
|
||||
options.one_euro_filter().min_allowed_object_scale(),
|
||||
options.one_euro_filter().disable_value_scaling());
|
||||
} else {
|
||||
RET_CHECK_FAIL()
|
||||
<< "Landmarks filter is either not specified or not supported";
|
||||
}
|
||||
}
|
||||
|
||||
absl::StatusOr<LandmarksFilter*> MultiLandmarkFilters::GetOrCreate(
|
||||
const int64_t tracking_id,
|
||||
const mediapipe::LandmarksSmoothingCalculatorOptions& options) {
|
||||
const auto it = filters_.find(tracking_id);
|
||||
if (it != filters_.end()) {
|
||||
return it->second.get();
|
||||
}
|
||||
|
||||
ASSIGN_OR_RETURN(auto landmarks_filter, InitializeLandmarksFilter(options));
|
||||
filters_[tracking_id] = std::move(landmarks_filter);
|
||||
return filters_[tracking_id].get();
|
||||
}
|
||||
|
||||
void MultiLandmarkFilters::ClearUnused(
|
||||
const std::vector<int64_t>& tracking_ids) {
|
||||
std::vector<int64_t> unused_tracking_ids;
|
||||
for (const auto& it : filters_) {
|
||||
bool unused = true;
|
||||
for (int64_t tracking_id : tracking_ids) {
|
||||
if (tracking_id == it.first) unused = false;
|
||||
}
|
||||
if (unused) unused_tracking_ids.push_back(it.first);
|
||||
}
|
||||
|
||||
for (int64_t tracking_id : unused_tracking_ids) {
|
||||
filters_.erase(tracking_id);
|
||||
}
|
||||
}
|
||||
|
||||
void MultiLandmarkFilters::Clear() { filters_.clear(); }
|
||||
|
||||
} // namespace landmarks_smoothing
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
|
||||
#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/util/filtering/one_euro_filter.h"
|
||||
#include "mediapipe/util/filtering/relative_velocity_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace landmarks_smoothing {
|
||||
|
||||
void NormalizedLandmarksToLandmarks(
|
||||
const mediapipe::NormalizedLandmarkList& norm_landmarks,
|
||||
const int image_width, const int image_height,
|
||||
mediapipe::LandmarkList& landmarks);
|
||||
|
||||
void LandmarksToNormalizedLandmarks(
|
||||
const mediapipe::LandmarkList& landmarks, const int image_width,
|
||||
const int image_height, mediapipe::NormalizedLandmarkList& norm_landmarks);
|
||||
|
||||
float GetObjectScale(const NormalizedRect& roi, const int image_width,
|
||||
const int image_height);
|
||||
|
||||
float GetObjectScale(const Rect& roi);
|
||||
|
||||
// Abstract class for various landmarks filters.
|
||||
class LandmarksFilter {
|
||||
public:
|
||||
virtual ~LandmarksFilter() = default;
|
||||
|
||||
virtual absl::Status Reset() { return absl::OkStatus(); }
|
||||
|
||||
virtual absl::Status Apply(const mediapipe::LandmarkList& in_landmarks,
|
||||
const absl::Duration& timestamp,
|
||||
const absl::optional<float> object_scale_opt,
|
||||
mediapipe::LandmarkList& out_landmarks) = 0;
|
||||
};
|
||||
|
||||
absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
|
||||
const mediapipe::LandmarksSmoothingCalculatorOptions& options);
|
||||
|
||||
class MultiLandmarkFilters {
|
||||
public:
|
||||
virtual ~MultiLandmarkFilters() = default;
|
||||
|
||||
virtual absl::StatusOr<LandmarksFilter*> GetOrCreate(
|
||||
const int64_t tracking_id,
|
||||
const mediapipe::LandmarksSmoothingCalculatorOptions& options);
|
||||
|
||||
virtual void ClearUnused(const std::vector<int64_t>& tracking_ids);
|
||||
|
||||
virtual void Clear();
|
||||
|
||||
private:
|
||||
std::map<int64_t, std::unique_ptr<LandmarksFilter>> filters_;
|
||||
};
|
||||
|
||||
} // namespace landmarks_smoothing
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
|
||||
@@ -0,0 +1,118 @@
|
||||
/* Copyright 2023 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace landmarks_smoothing {
|
||||
namespace {
|
||||
|
||||
TEST(LandmarksSmoothingCalculatorUtilsTest, NormalizedLandmarksToLandmarks) {
|
||||
NormalizedLandmarkList norm_landmarks;
|
||||
NormalizedLandmark* norm_landmark = norm_landmarks.add_landmark();
|
||||
norm_landmark->set_x(0.1);
|
||||
norm_landmark->set_y(0.2);
|
||||
norm_landmark->set_z(0.3);
|
||||
norm_landmark->set_visibility(0.4);
|
||||
norm_landmark->set_presence(0.5);
|
||||
|
||||
LandmarkList landmarks;
|
||||
NormalizedLandmarksToLandmarks(norm_landmarks, /*image_width=*/10,
|
||||
/*image_height=*/10, landmarks);
|
||||
|
||||
EXPECT_EQ(landmarks.landmark_size(), 1);
|
||||
Landmark landmark = landmarks.landmark(0);
|
||||
EXPECT_NEAR(landmark.x(), 1.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.y(), 2.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.z(), 3.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.visibility(), 0.4, 1e-6);
|
||||
EXPECT_NEAR(landmark.presence(), 0.5, 1e-6);
|
||||
}
|
||||
|
||||
TEST(LandmarksSmoothingCalculatorUtilsTest,
|
||||
NormalizedLandmarksToLandmarks_EmptyVisibilityAndPresence) {
|
||||
NormalizedLandmarkList norm_landmarks;
|
||||
NormalizedLandmark* norm_landmark = norm_landmarks.add_landmark();
|
||||
norm_landmark->set_x(0.1);
|
||||
norm_landmark->set_y(0.2);
|
||||
norm_landmark->set_z(0.3);
|
||||
norm_landmark->clear_visibility();
|
||||
norm_landmark->clear_presence();
|
||||
|
||||
LandmarkList landmarks;
|
||||
NormalizedLandmarksToLandmarks(norm_landmarks, /*image_width=*/10,
|
||||
/*image_height=*/10, landmarks);
|
||||
|
||||
EXPECT_EQ(landmarks.landmark_size(), 1);
|
||||
Landmark landmark = landmarks.landmark(0);
|
||||
EXPECT_NEAR(landmark.x(), 1.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.y(), 2.0, 1e-6);
|
||||
EXPECT_NEAR(landmark.z(), 3.0, 1e-6);
|
||||
EXPECT_FALSE(landmark.has_visibility());
|
||||
EXPECT_FALSE(landmark.has_presence());
|
||||
}
|
||||
|
||||
TEST(LandmarksSmoothingCalculatorUtilsTest, LandmarksToNormalizedLandmarks) {
|
||||
LandmarkList landmarks;
|
||||
Landmark* landmark = landmarks.add_landmark();
|
||||
landmark->set_x(1.0);
|
||||
landmark->set_y(2.0);
|
||||
landmark->set_z(3.0);
|
||||
landmark->set_visibility(0.4);
|
||||
landmark->set_presence(0.5);
|
||||
|
||||
NormalizedLandmarkList norm_landmarks;
|
||||
LandmarksToNormalizedLandmarks(landmarks, /*image_width=*/10,
|
||||
/*image_height=*/10, norm_landmarks);
|
||||
|
||||
EXPECT_EQ(norm_landmarks.landmark_size(), 1);
|
||||
NormalizedLandmark norm_landmark = norm_landmarks.landmark(0);
|
||||
EXPECT_NEAR(norm_landmark.x(), 0.1, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.y(), 0.2, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.z(), 0.3, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.visibility(), 0.4, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.presence(), 0.5, 1e-6);
|
||||
}
|
||||
|
||||
TEST(LandmarksSmoothingCalculatorUtilsTest,
|
||||
LandmarksToNormalizedLandmarks_EmptyVisibilityAndPresence) {
|
||||
LandmarkList landmarks;
|
||||
Landmark* landmark = landmarks.add_landmark();
|
||||
landmark->set_x(1.0);
|
||||
landmark->set_y(2.0);
|
||||
landmark->set_z(3.0);
|
||||
landmark->clear_visibility();
|
||||
landmark->clear_presence();
|
||||
|
||||
NormalizedLandmarkList norm_landmarks;
|
||||
LandmarksToNormalizedLandmarks(landmarks, /*image_width=*/10,
|
||||
/*image_height=*/10, norm_landmarks);
|
||||
|
||||
EXPECT_EQ(norm_landmarks.landmark_size(), 1);
|
||||
NormalizedLandmark norm_landmark = norm_landmarks.landmark(0);
|
||||
EXPECT_NEAR(norm_landmark.x(), 0.1, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.y(), 0.2, 1e-6);
|
||||
EXPECT_NEAR(norm_landmark.z(), 0.3, 1e-6);
|
||||
EXPECT_FALSE(norm_landmark.has_visibility());
|
||||
EXPECT_FALSE(norm_landmark.has_presence());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace landmarks_smoothing
|
||||
} // namespace mediapipe
|
||||
@@ -18,6 +18,9 @@ package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
option java_package = "com.google.mediapipe.calculator.proto";
|
||||
option java_outer_classname = "LogicCalculatorOptionsProto";
|
||||
|
||||
message LogicCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional LogicCalculatorOptions ext = 338731246;
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/util/multi_landmarks_smoothing_calculator.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
namespace {
|
||||
|
||||
using ::mediapipe::NormalizedRect;
|
||||
using ::mediapipe::landmarks_smoothing::GetObjectScale;
|
||||
using ::mediapipe::landmarks_smoothing::LandmarksToNormalizedLandmarks;
|
||||
using ::mediapipe::landmarks_smoothing::MultiLandmarkFilters;
|
||||
using ::mediapipe::landmarks_smoothing::NormalizedLandmarksToLandmarks;
|
||||
|
||||
} // namespace
|
||||
|
||||
class MultiLandmarksSmoothingCalculatorImpl
|
||||
: public NodeImpl<MultiLandmarksSmoothingCalculator> {
|
||||
public:
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if (kInNormLandmarks(cc).IsEmpty()) {
|
||||
multi_filters_.Clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
const auto& tracking_ids = kTrackingIds(cc).Get();
|
||||
multi_filters_.ClearUnused(tracking_ids);
|
||||
|
||||
const auto& in_norm_landmarks_vec = kInNormLandmarks(cc).Get();
|
||||
RET_CHECK_EQ(in_norm_landmarks_vec.size(), tracking_ids.size());
|
||||
|
||||
int image_width;
|
||||
int image_height;
|
||||
std::tie(image_width, image_height) = kImageSize(cc).Get();
|
||||
|
||||
std::optional<std::vector<NormalizedRect>> object_scale_roi_vec;
|
||||
if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
|
||||
object_scale_roi_vec = kObjectScaleRoi(cc).Get();
|
||||
RET_CHECK_EQ(object_scale_roi_vec.value().size(), tracking_ids.size());
|
||||
}
|
||||
|
||||
std::vector<NormalizedLandmarkList> out_norm_landmarks_vec;
|
||||
for (int i = 0; i < tracking_ids.size(); ++i) {
|
||||
LandmarkList in_landmarks;
|
||||
NormalizedLandmarksToLandmarks(in_norm_landmarks_vec[i], image_width,
|
||||
image_height, in_landmarks);
|
||||
|
||||
std::optional<float> object_scale;
|
||||
if (object_scale_roi_vec) {
|
||||
object_scale = GetObjectScale(object_scale_roi_vec.value()[i],
|
||||
image_width, image_height);
|
||||
}
|
||||
|
||||
ASSIGN_OR_RETURN(auto* landmarks_filter,
|
||||
multi_filters_.GetOrCreate(
|
||||
tracking_ids[i],
|
||||
cc->Options<LandmarksSmoothingCalculatorOptions>()));
|
||||
|
||||
LandmarkList out_landmarks;
|
||||
MP_RETURN_IF_ERROR(landmarks_filter->Apply(in_landmarks, timestamp,
|
||||
object_scale, out_landmarks));
|
||||
|
||||
NormalizedLandmarkList out_norm_landmarks;
|
||||
LandmarksToNormalizedLandmarks(out_landmarks, image_width, image_height,
|
||||
out_norm_landmarks);
|
||||
|
||||
out_norm_landmarks_vec.push_back(std::move(out_norm_landmarks));
|
||||
}
|
||||
|
||||
kOutNormLandmarks(cc).Send(std::move(out_norm_landmarks_vec));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
MultiLandmarkFilters multi_filters_;
|
||||
};
|
||||
MEDIAPIPE_NODE_IMPLEMENTATION(MultiLandmarksSmoothingCalculatorImpl);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
//
|
||||
// Inputs:
|
||||
// NORM_LANDMARKS: A std::vector<NormalizedLandmarkList> of landmarks you want
|
||||
// to smooth.
|
||||
// TRACKING_IDS: A std<int64_t> vector of tracking IDs used to associate
|
||||
// landmarks over time. When new ID arrives - calculator will initialize new
|
||||
// filter. When tracking ID is no longer provided - calculator will forget
|
||||
// smoothing state.
|
||||
// IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
|
||||
// Required to perform all computations in absolute coordinates to avoid any
|
||||
// influence of normalized values.
|
||||
// OBJECT_SCALE_ROI (optional): A std::vector<NormRect> used to determine the
|
||||
// object scale for some of the filters. If not provided - object scale will
|
||||
// be calculated from landmarks.
|
||||
//
|
||||
// Outputs:
|
||||
// NORM_FILTERED_LANDMARKS: A std::vector<NormalizedLandmarkList> of smoothed
|
||||
// landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "MultiLandmarksSmoothingCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:pose_landmarks"
|
||||
// input_stream: "IMAGE_SIZE:image_size"
|
||||
// input_stream: "OBJECT_SCALE_ROI:roi"
|
||||
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
|
||||
// options: {
|
||||
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
// velocity_filter: {
|
||||
// window_size: 5
|
||||
// velocity_scale: 10.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class MultiLandmarksSmoothingCalculator : public NodeIntf {
|
||||
public:
|
||||
static constexpr Input<std::vector<mediapipe::NormalizedLandmarkList>>
|
||||
kInNormLandmarks{"NORM_LANDMARKS"};
|
||||
static constexpr Input<std::vector<int64_t>> kTrackingIds{"TRACKING_IDS"};
|
||||
static constexpr Input<std::pair<int, int>> kImageSize{"IMAGE_SIZE"};
|
||||
static constexpr Input<std::vector<NormalizedRect>>::Optional kObjectScaleRoi{
|
||||
"OBJECT_SCALE_ROI"};
|
||||
static constexpr Output<std::vector<mediapipe::NormalizedLandmarkList>>
|
||||
kOutNormLandmarks{"NORM_FILTERED_LANDMARKS"};
|
||||
|
||||
MEDIAPIPE_NODE_INTERFACE(MultiLandmarksSmoothingCalculator, kInNormLandmarks,
|
||||
kTrackingIds, kImageSize, kObjectScaleRoi,
|
||||
kOutNormLandmarks);
|
||||
};
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/util/multi_world_landmarks_smoothing_calculator.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
namespace {
|
||||
|
||||
using ::mediapipe::Rect;
|
||||
using ::mediapipe::landmarks_smoothing::GetObjectScale;
|
||||
using ::mediapipe::landmarks_smoothing::MultiLandmarkFilters;
|
||||
|
||||
} // namespace
|
||||
|
||||
class MultiWorldLandmarksSmoothingCalculatorImpl
|
||||
: public NodeImpl<MultiWorldLandmarksSmoothingCalculator> {
|
||||
public:
|
||||
absl::Status Process(CalculatorContext* cc) override {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if (kInLandmarks(cc).IsEmpty()) {
|
||||
multi_filters_.Clear();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
const auto& tracking_ids = kTrackingIds(cc).Get();
|
||||
multi_filters_.ClearUnused(tracking_ids);
|
||||
|
||||
const auto& in_landmarks_vec = kInLandmarks(cc).Get();
|
||||
RET_CHECK_EQ(in_landmarks_vec.size(), tracking_ids.size());
|
||||
|
||||
std::optional<std::vector<Rect>> object_scale_roi_vec;
|
||||
if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
|
||||
object_scale_roi_vec = kObjectScaleRoi(cc).Get();
|
||||
RET_CHECK_EQ(object_scale_roi_vec.value().size(), tracking_ids.size());
|
||||
}
|
||||
|
||||
std::vector<LandmarkList> out_landmarks_vec;
|
||||
for (int i = 0; i < tracking_ids.size(); ++i) {
|
||||
const auto& in_landmarks = in_landmarks_vec[i];
|
||||
|
||||
std::optional<float> object_scale;
|
||||
if (object_scale_roi_vec) {
|
||||
object_scale = GetObjectScale(object_scale_roi_vec.value()[i]);
|
||||
}
|
||||
|
||||
ASSIGN_OR_RETURN(auto* landmarks_filter,
|
||||
multi_filters_.GetOrCreate(
|
||||
tracking_ids[i],
|
||||
cc->Options<LandmarksSmoothingCalculatorOptions>()));
|
||||
|
||||
LandmarkList out_landmarks;
|
||||
MP_RETURN_IF_ERROR(landmarks_filter->Apply(in_landmarks, timestamp,
|
||||
object_scale, out_landmarks));
|
||||
|
||||
out_landmarks_vec.push_back(std::move(out_landmarks));
|
||||
}
|
||||
|
||||
kOutLandmarks(cc).Send(std::move(out_landmarks_vec));
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
MultiLandmarkFilters multi_filters_;
|
||||
};
|
||||
MEDIAPIPE_NODE_IMPLEMENTATION(MultiWorldLandmarksSmoothingCalculatorImpl);
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
#define MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
|
||||
#include "mediapipe/framework/api2/node.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace api2 {
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
//
|
||||
// Inputs:
|
||||
// LANDMARKS: A std::vector<LandmarkList> of landmarks you want to
|
||||
// smooth.
|
||||
// TRACKING_IDS: A std<int64_t> vector of tracking IDs used to associate
|
||||
// landmarks over time. When new ID arrives - calculator will initialize new
|
||||
// filter. When tracking ID is no longer provided - calculator will forget
|
||||
// smoothing state.
|
||||
// OBJECT_SCALE_ROI (optional): A std::vector<Rect> used to determine the
|
||||
// object scale for some of the filters. If not provided - object scale will
|
||||
// be calculated from landmarks.
|
||||
//
|
||||
// Outputs:
|
||||
// FILTERED_LANDMARKS: A std::vector<LandmarkList> of smoothed landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "MultiWorldLandmarksSmoothingCalculator"
|
||||
// input_stream: "LANDMARKS:landmarks"
|
||||
// input_stream: "OBJECT_SCALE_ROI:roi"
|
||||
// output_stream: "FILTERED_LANDMARKS:landmarks_filtered"
|
||||
// options: {
|
||||
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
// velocity_filter: {
|
||||
// window_size: 5
|
||||
// velocity_scale: 10.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class MultiWorldLandmarksSmoothingCalculator : public NodeIntf {
|
||||
public:
|
||||
static constexpr Input<std::vector<mediapipe::LandmarkList>> kInLandmarks{
|
||||
"LANDMARKS"};
|
||||
static constexpr Input<std::vector<int64_t>> kTrackingIds{"TRACKING_IDS"};
|
||||
static constexpr Input<std::vector<Rect>>::Optional kObjectScaleRoi{
|
||||
"OBJECT_SCALE_ROI"};
|
||||
static constexpr Output<std::vector<mediapipe::LandmarkList>> kOutLandmarks{
|
||||
"FILTERED_LANDMARKS"};
|
||||
|
||||
MEDIAPIPE_NODE_INTERFACE(MultiWorldLandmarksSmoothingCalculator, kInLandmarks,
|
||||
kTrackingIds, kObjectScaleRoi, kOutLandmarks);
|
||||
};
|
||||
|
||||
} // namespace api2
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
|
||||
@@ -48,6 +48,7 @@ cc_library(
|
||||
"//mediapipe/framework/port:opencv_video",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/util:resource_util",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/flags:parse",
|
||||
],
|
||||
@@ -73,6 +74,7 @@ cc_library(
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_shared_data_internal",
|
||||
"//mediapipe/util:resource_util",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/flags:parse",
|
||||
],
|
||||
|
||||
@@ -28,11 +28,8 @@
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
using mediapipe::Adopt;
|
||||
using mediapipe::CalculatorBase;
|
||||
using mediapipe::ImageFrame;
|
||||
using mediapipe::PacketTypeSet;
|
||||
using mediapipe::autoflip::Border;
|
||||
|
||||
constexpr char kDetectedBorders[] = "DETECTED_BORDERS";
|
||||
constexpr int kMinBorderDistance = 5;
|
||||
|
||||
@@ -28,16 +28,12 @@
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
using mediapipe::Adopt;
|
||||
using mediapipe::CalculatorGraphConfig;
|
||||
using mediapipe::CalculatorRunner;
|
||||
using mediapipe::ImageFormat;
|
||||
using mediapipe::ImageFrame;
|
||||
using mediapipe::Packet;
|
||||
using mediapipe::PacketTypeSet;
|
||||
using mediapipe::ParseTextProtoOrDie;
|
||||
using mediapipe::Timestamp;
|
||||
using mediapipe::autoflip::Border;
|
||||
|
||||
namespace mediapipe {
|
||||
namespace autoflip {
|
||||
|
||||
@@ -31,14 +31,11 @@
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
using mediapipe::Adopt;
|
||||
using mediapipe::CalculatorGraphConfig;
|
||||
using mediapipe::CalculatorRunner;
|
||||
using mediapipe::ImageFormat;
|
||||
using mediapipe::ImageFrame;
|
||||
using mediapipe::PacketTypeSet;
|
||||
using mediapipe::ParseTextProtoOrDie;
|
||||
using mediapipe::Timestamp;
|
||||
|
||||
namespace mediapipe {
|
||||
namespace autoflip {
|
||||
|
||||
@@ -28,8 +28,6 @@
|
||||
using mediapipe::Packet;
|
||||
using mediapipe::PacketTypeSet;
|
||||
using mediapipe::autoflip::DetectionSet;
|
||||
using mediapipe::autoflip::SalientRegion;
|
||||
using mediapipe::autoflip::SignalType;
|
||||
|
||||
constexpr char kIsShotBoundaryTag[] = "IS_SHOT_BOUNDARY";
|
||||
constexpr char kSignalInputsTag[] = "SIGNAL";
|
||||
|
||||
@@ -190,14 +190,16 @@ TEST(PaddingEffectGeneratorTest, ScaleToMultipleOfTwo) {
|
||||
double target_aspect_ratio = 0.5;
|
||||
int expect_width = 14;
|
||||
int expect_height = input_height;
|
||||
auto test_frame = absl::make_unique<ImageFrame>(/*format=*/ImageFormat::SRGB,
|
||||
input_width, input_height);
|
||||
ImageFrame test_frame(/*format=*/ImageFormat::SRGB, input_width,
|
||||
input_height);
|
||||
cv::Mat mat = formats::MatView(&test_frame);
|
||||
mat = cv::Scalar(0, 0, 0);
|
||||
|
||||
PaddingEffectGenerator generator(test_frame->Width(), test_frame->Height(),
|
||||
PaddingEffectGenerator generator(test_frame.Width(), test_frame.Height(),
|
||||
target_aspect_ratio,
|
||||
/*scale_to_multiple_of_two=*/true);
|
||||
ImageFrame result_frame;
|
||||
MP_ASSERT_OK(generator.Process(*test_frame, 0.3, 40, 0.0, &result_frame));
|
||||
MP_ASSERT_OK(generator.Process(test_frame, 0.3, 40, 0.0, &result_frame));
|
||||
EXPECT_EQ(result_frame.Width(), expect_width);
|
||||
EXPECT_EQ(result_frame.Height(), expect_height);
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
#include "mediapipe/framework/port/opencv_video_inc.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
|
||||
constexpr char kInputStream[] = "input_video";
|
||||
constexpr char kOutputStream[] = "output_video";
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "mediapipe/gpu/gpu_shared_data_internal.h"
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
|
||||
constexpr char kInputStream[] = "input_video";
|
||||
constexpr char kOutputStream[] = "output_video";
|
||||
|
||||
@@ -147,12 +147,18 @@ def main():
|
||||
f"Looking for profiles for app ids with prefix '{bundle_id_prefix}' in '{profile_dir}'"
|
||||
)
|
||||
|
||||
profiles_found = False
|
||||
for name in os.listdir(profile_dir):
|
||||
if not name.endswith(".mobileprovision"):
|
||||
continue
|
||||
profiles_found = True
|
||||
profile_path = os.path.join(profile_dir, name)
|
||||
process_profile(profile_path, our_app_id_re)
|
||||
|
||||
if not profiles_found:
|
||||
print("Error: Unable to find any provisioning profiles " +
|
||||
f"(*.mobileprovision files) in '{profile_dir}'")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -33,7 +33,9 @@ bzl_library(
|
||||
srcs = [
|
||||
"transitive_protos.bzl",
|
||||
],
|
||||
visibility = ["//mediapipe/framework:__subpackages__"],
|
||||
visibility = [
|
||||
"//mediapipe/framework:__subpackages__",
|
||||
],
|
||||
)
|
||||
|
||||
bzl_library(
|
||||
@@ -1097,6 +1099,7 @@ cc_library(
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = True, # Defines TestServiceCalculator
|
||||
)
|
||||
|
||||
cc_library(
|
||||
@@ -1403,6 +1406,7 @@ cc_test(
|
||||
"calculator_graph_test.cc",
|
||||
],
|
||||
linkstatic = 1,
|
||||
tags = ["not_run:arm"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":calculator_framework",
|
||||
|
||||
@@ -52,7 +52,7 @@ int select = cc->Inputs().Tag(kSelectTag).Get<int>();
|
||||
write
|
||||
|
||||
```
|
||||
int select = kSelectTag(cc).Get(); // alternative: *kSelectTag(cc)
|
||||
int select = kSelect(cc).Get(); // alternative: *kSelect(cc)
|
||||
```
|
||||
|
||||
Sets of multiple ports can be declared with `::Multiple`. Note, also, that a tag
|
||||
|
||||
@@ -223,6 +223,16 @@ class SourceImpl {
|
||||
return !(*this == other);
|
||||
}
|
||||
|
||||
Src& SetName(const char* name) {
|
||||
base_->name_ = std::string(name);
|
||||
return *this;
|
||||
}
|
||||
|
||||
Src& SetName(absl::string_view name) {
|
||||
base_->name_ = std::string(name);
|
||||
return *this;
|
||||
}
|
||||
|
||||
Src& SetName(std::string name) {
|
||||
base_->name_ = std::move(name);
|
||||
return *this;
|
||||
|
||||
@@ -19,8 +19,6 @@ namespace mediapipe {
|
||||
namespace api2 {
|
||||
namespace test {
|
||||
|
||||
using testing::ElementsAre;
|
||||
|
||||
// Returns the packet values for a vector of Packets.
|
||||
template <typename T>
|
||||
std::vector<T> PacketValues(const std::vector<mediapipe::Packet>& packets) {
|
||||
|
||||
@@ -165,7 +165,7 @@ template <class V, class... U>
|
||||
struct IsCompatibleType<V, OneOf<U...>>
|
||||
: std::integral_constant<bool, (std::is_same_v<V, U> || ...)> {};
|
||||
|
||||
}; // namespace internal
|
||||
} // namespace internal
|
||||
|
||||
template <typename T>
|
||||
inline Packet<T> PacketBase::As() const {
|
||||
@@ -259,19 +259,19 @@ struct First {
|
||||
|
||||
template <class T>
|
||||
struct AddStatus {
|
||||
using type = StatusOr<T>;
|
||||
using type = absl::StatusOr<T>;
|
||||
};
|
||||
template <class T>
|
||||
struct AddStatus<StatusOr<T>> {
|
||||
using type = StatusOr<T>;
|
||||
struct AddStatus<absl::StatusOr<T>> {
|
||||
using type = absl::StatusOr<T>;
|
||||
};
|
||||
template <>
|
||||
struct AddStatus<Status> {
|
||||
using type = Status;
|
||||
struct AddStatus<absl::Status> {
|
||||
using type = absl::Status;
|
||||
};
|
||||
template <>
|
||||
struct AddStatus<void> {
|
||||
using type = Status;
|
||||
using type = absl::Status;
|
||||
};
|
||||
|
||||
template <class R, class F, class... A>
|
||||
@@ -282,7 +282,7 @@ struct CallAndAddStatusImpl {
|
||||
};
|
||||
template <class F, class... A>
|
||||
struct CallAndAddStatusImpl<void, F, A...> {
|
||||
Status operator()(const F& f, A&&... a) {
|
||||
absl::Status operator()(const F& f, A&&... a) {
|
||||
f(std::forward<A>(a)...);
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -467,6 +467,11 @@ class SideFallbackT : public Base {
|
||||
// CalculatorContext (e.g. kOut(cc)), and provides a type-safe interface to
|
||||
// OutputStreamShard. Like that class, this class will not be usually named in
|
||||
// calculator code, but used as a temporary object (e.g. kOut(cc).Send(...)).
|
||||
//
|
||||
// If not connected (!IsConnected()) SetNextTimestampBound is safe to call and
|
||||
// does nothing.
|
||||
// All the sub-classes that define Send should implement it to be safe to to
|
||||
// call if not connected and do nothing in such case.
|
||||
class OutputShardAccessBase {
|
||||
public:
|
||||
OutputShardAccessBase(const CalculatorContext& cc, OutputStreamShard* output)
|
||||
|
||||
@@ -23,15 +23,13 @@ package mediapipe;
|
||||
option java_package = "com.google.mediapipe.proto";
|
||||
option java_outer_classname = "CalculatorOptionsProto";
|
||||
|
||||
// Options for Calculators. Each Calculator implementation should
|
||||
// have its own options proto, which should look like this:
|
||||
// Options for Calculators, DEPRECATED. New calculators are encouraged to use
|
||||
// proto3 syntax options:
|
||||
//
|
||||
// message MyCalculatorOptions {
|
||||
// extend CalculatorOptions {
|
||||
// optional MyCalculatorOptions ext = <unique id, e.g. the CL#>;
|
||||
// }
|
||||
// optional string field_needed_by_my_calculator = 1;
|
||||
// optional int32 another_field = 2;
|
||||
// // proto3 does not expect "optional"
|
||||
// string field_needed_by_my_calculator = 1;
|
||||
// int32 another_field = 2;
|
||||
// // etc
|
||||
// }
|
||||
message CalculatorOptions {
|
||||
|
||||
@@ -88,10 +88,13 @@ class SafeIntStrongIntValidator {
|
||||
|
||||
// If the argument is floating point, we can do a simple check to make
|
||||
// sure the value is in range. It is undefined behavior to convert to int
|
||||
// from a float that is out of range.
|
||||
// from a float that is out of range. Since large integers will loose some
|
||||
// precision when being converted to floating point, the integer max and min
|
||||
// are explicitly converted back to floating point for this comparison, in
|
||||
// order to satisfy compiler warnings.
|
||||
if (std::is_floating_point<U>::value) {
|
||||
if (arg < std::numeric_limits<T>::min() ||
|
||||
arg > std::numeric_limits<T>::max()) {
|
||||
if (arg < static_cast<U>(std::numeric_limits<T>::min()) ||
|
||||
arg > static_cast<U>(std::numeric_limits<T>::max())) {
|
||||
ErrorType::Error("SafeInt: init from out of bounds float", arg, "=");
|
||||
}
|
||||
} else {
|
||||
@@ -284,11 +287,11 @@ class SafeIntStrongIntValidator {
|
||||
// A SafeIntStrongIntValidator policy class to LOG(FATAL) on errors.
|
||||
struct LogFatalOnError {
|
||||
template <typename Tlhs, typename Trhs>
|
||||
static void Error(const char *error, Tlhs lhs, Trhs rhs, const char *op) {
|
||||
static void Error(const char* error, Tlhs lhs, Trhs rhs, const char* op) {
|
||||
LOG(FATAL) << error << ": (" << lhs << " " << op << " " << rhs << ")";
|
||||
}
|
||||
template <typename Tval>
|
||||
static void Error(const char *error, Tval val, const char *op) {
|
||||
static void Error(const char* error, Tval val, const char* op) {
|
||||
LOG(FATAL) << error << ": (" << op << val << ")";
|
||||
}
|
||||
};
|
||||
|
||||
@@ -68,11 +68,11 @@ StatusBuilder&& StatusBuilder::SetNoLogging() && {
|
||||
return std::move(SetNoLogging());
|
||||
}
|
||||
|
||||
StatusBuilder::operator Status() const& {
|
||||
StatusBuilder::operator absl::Status() const& {
|
||||
return StatusBuilder(*this).JoinMessageToStatus();
|
||||
}
|
||||
|
||||
StatusBuilder::operator Status() && { return JoinMessageToStatus(); }
|
||||
StatusBuilder::operator absl::Status() && { return JoinMessageToStatus(); }
|
||||
|
||||
absl::Status StatusBuilder::JoinMessageToStatus() {
|
||||
if (!impl_) {
|
||||
|
||||
@@ -83,8 +83,8 @@ class ABSL_MUST_USE_RESULT StatusBuilder {
|
||||
return std::move(*this << msg);
|
||||
}
|
||||
|
||||
operator Status() const&;
|
||||
operator Status() &&;
|
||||
operator absl::Status() const&;
|
||||
operator absl::Status() &&;
|
||||
|
||||
absl::Status JoinMessageToStatus();
|
||||
|
||||
|
||||
@@ -403,11 +403,11 @@ std::ostream &operator<<(std::ostream &os,
|
||||
lhs op## = rhs; \
|
||||
return lhs; \
|
||||
}
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(+);
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(-);
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(&);
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(|);
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(^);
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(+)
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(-)
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(&)
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(|)
|
||||
STRONG_INT_VS_STRONG_INT_BINARY_OP(^)
|
||||
#undef STRONG_INT_VS_STRONG_INT_BINARY_OP
|
||||
|
||||
// Define operators that take one StrongInt and one native integer argument.
|
||||
@@ -431,12 +431,12 @@ STRONG_INT_VS_STRONG_INT_BINARY_OP(^);
|
||||
rhs op## = lhs; \
|
||||
return rhs; \
|
||||
}
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(*);
|
||||
NUMERIC_VS_STRONG_INT_BINARY_OP(*);
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(/);
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(%);
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(<<); // NOLINT(whitespace/operators)
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(>>); // NOLINT(whitespace/operators)
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(*)
|
||||
NUMERIC_VS_STRONG_INT_BINARY_OP(*)
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(/)
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(%)
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(<<) // NOLINT(whitespace/operators)
|
||||
STRONG_INT_VS_NUMERIC_BINARY_OP(>>) // NOLINT(whitespace/operators)
|
||||
#undef STRONG_INT_VS_NUMERIC_BINARY_OP
|
||||
#undef NUMERIC_VS_STRONG_INT_BINARY_OP
|
||||
|
||||
@@ -447,12 +447,12 @@ STRONG_INT_VS_NUMERIC_BINARY_OP(>>); // NOLINT(whitespace/operators)
|
||||
StrongInt<TagType, ValueType, ValidatorType> rhs) { \
|
||||
return lhs.value() op rhs.value(); \
|
||||
}
|
||||
STRONG_INT_COMPARISON_OP(==); // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(!=); // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(<); // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(<=); // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(>); // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(>=); // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(==) // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(!=) // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(<) // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(<=) // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(>) // NOLINT(whitespace/operators)
|
||||
STRONG_INT_COMPARISON_OP(>=) // NOLINT(whitespace/operators)
|
||||
#undef STRONG_INT_COMPARISON_OP
|
||||
|
||||
} // namespace intops
|
||||
|
||||
@@ -481,6 +481,7 @@ cc_library(
|
||||
cc_test(
|
||||
name = "tensor_test",
|
||||
srcs = ["tensor_test.cc"],
|
||||
tags = ["not_run:arm"],
|
||||
deps = [
|
||||
":tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2023 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
||||
/* Copyright 2023 The MediaPipe Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -113,11 +113,11 @@ class Image {
|
||||
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
// Get a GPU view. Automatically uploads from CPU if needed.
|
||||
const mediapipe::GpuBuffer GetGpuBuffer() const {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (use_gpu_ == false) ConvertToGpu();
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
// Provides access to the underlying GpuBuffer storage.
|
||||
// Automatically uploads from CPU to GPU if needed and requested through the
|
||||
// `upload_to_gpu` argument.
|
||||
const mediapipe::GpuBuffer GetGpuBuffer(bool upload_to_gpu = true) const {
|
||||
if (!use_gpu_ && upload_to_gpu) ConvertToGpu();
|
||||
return gpu_buffer_;
|
||||
}
|
||||
|
||||
|
||||
@@ -117,8 +117,8 @@ MtlBufferView MtlBufferView::GetReadView(const Tensor& tensor,
|
||||
<< "Tensor must be written prior to read from.";
|
||||
LOG_IF(FATAL,
|
||||
!(tensor.valid_ & (Tensor::kValidCpu | Tensor::kValidMetalBuffer)))
|
||||
<< "Tensor conversion between different GPU resources is not supported "
|
||||
"yet.";
|
||||
<< "Tensor conversion between different GPU backing formats is not "
|
||||
"supported yet.";
|
||||
auto lock(absl::make_unique<absl::MutexLock>(&tensor.view_mutex_));
|
||||
tensor.valid_ |= Tensor::kValidMetalBuffer;
|
||||
AllocateMtlBuffer(tensor, [command_buffer device]);
|
||||
@@ -164,8 +164,8 @@ Tensor::OpenGlTexture2dView Tensor::GetOpenGlTexture2dReadView() const {
|
||||
LOG_IF(FATAL, valid_ == kValidNone)
|
||||
<< "Tensor must be written prior to read from.";
|
||||
LOG_IF(FATAL, !(valid_ & (kValidCpu | kValidOpenGlTexture2d)))
|
||||
<< "Tensor conversion between different GPU resources is not supported "
|
||||
"yet.";
|
||||
<< "Tensor conversion between different GPU backing formats is not "
|
||||
"supported yet.";
|
||||
auto lock = absl::make_unique<absl::MutexLock>(&view_mutex_);
|
||||
AllocateOpenGlTexture2d();
|
||||
if (!(valid_ & kValidOpenGlTexture2d)) {
|
||||
@@ -335,7 +335,8 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferReadView() const {
|
||||
kValidAHardwareBuffer |
|
||||
#endif // MEDIAPIPE_TENSOR_USE_AHWB
|
||||
kValidOpenGlBuffer)))
|
||||
<< "Tensor conversion between different GPU resources is not supported.";
|
||||
<< "Tensor conversion between different GPU backing formats is not "
|
||||
"supported yet.";
|
||||
auto lock(absl::make_unique<absl::MutexLock>(&view_mutex_));
|
||||
AllocateOpenGlBuffer();
|
||||
if (!(valid_ & kValidOpenGlBuffer)) {
|
||||
|
||||
@@ -44,7 +44,6 @@ class GraphServiceBase {
|
||||
|
||||
constexpr GraphServiceBase(const char* key) : key(key) {}
|
||||
|
||||
virtual ~GraphServiceBase() = default;
|
||||
inline virtual absl::StatusOr<Packet> CreateDefaultObject() const {
|
||||
return DefaultInitializationUnsupported();
|
||||
}
|
||||
@@ -52,14 +51,32 @@ class GraphServiceBase {
|
||||
const char* key;
|
||||
|
||||
protected:
|
||||
// `GraphService<T>` objects, deriving `GraphServiceBase` are designed to be
|
||||
// global constants and not ever deleted through `GraphServiceBase`. Hence,
|
||||
// protected and non-virtual destructor which helps to make `GraphService<T>`
|
||||
// trivially destructible and properly defined as global constants.
|
||||
//
|
||||
// A class with any virtual functions should have a destructor that is either
|
||||
// public and virtual or else protected and non-virtual.
|
||||
// https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rc-dtor-virtual
|
||||
~GraphServiceBase() = default;
|
||||
|
||||
absl::Status DefaultInitializationUnsupported() const {
|
||||
return absl::UnimplementedError(absl::StrCat(
|
||||
"Graph service '", key, "' does not support default initialization"));
|
||||
}
|
||||
};
|
||||
|
||||
// A global constant to refer a service:
|
||||
// - Requesting `CalculatorContract::UseService` from calculator
|
||||
// - Accessing `Calculator/SubgraphContext::Service`from calculator/subgraph
|
||||
// - Setting before graph initialization `CalculatorGraph::SetServiceObject`
|
||||
//
|
||||
// NOTE: In headers, define your graph service reference safely as following:
|
||||
// `inline constexpr GraphService<YourService> kYourService("YourService");`
|
||||
//
|
||||
template <typename T>
|
||||
class GraphService : public GraphServiceBase {
|
||||
class GraphService final : public GraphServiceBase {
|
||||
public:
|
||||
using type = T;
|
||||
using packet_type = std::shared_ptr<T>;
|
||||
@@ -68,7 +85,7 @@ class GraphService : public GraphServiceBase {
|
||||
kDisallowDefaultInitialization)
|
||||
: GraphServiceBase(my_key), default_init_(default_init) {}
|
||||
|
||||
absl::StatusOr<Packet> CreateDefaultObject() const override {
|
||||
absl::StatusOr<Packet> CreateDefaultObject() const final {
|
||||
if (default_init_ != kAllowDefaultInitialization) {
|
||||
return DefaultInitializationUnsupported();
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
const GraphService<int> kIntService("mediapipe::IntService");
|
||||
constexpr GraphService<int> kIntService("mediapipe::IntService");
|
||||
} // namespace
|
||||
|
||||
TEST(GraphServiceManager, SetGetServiceObject) {
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
|
||||
#include "mediapipe/framework/graph_service.h"
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/canonical_errors.h"
|
||||
@@ -159,7 +161,7 @@ TEST_F(GraphServiceTest, CreateDefault) {
|
||||
|
||||
struct TestServiceData {};
|
||||
|
||||
const GraphService<TestServiceData> kTestServiceAllowDefaultInitialization(
|
||||
constexpr GraphService<TestServiceData> kTestServiceAllowDefaultInitialization(
|
||||
"kTestServiceAllowDefaultInitialization",
|
||||
GraphServiceBase::kAllowDefaultInitialization);
|
||||
|
||||
@@ -272,9 +274,13 @@ TEST(AllowDefaultInitializationGraphServiceTest,
|
||||
HasSubstr("Service is unavailable.")));
|
||||
}
|
||||
|
||||
const GraphService<TestServiceData> kTestServiceDisallowDefaultInitialization(
|
||||
"kTestServiceDisallowDefaultInitialization",
|
||||
GraphServiceBase::kDisallowDefaultInitialization);
|
||||
constexpr GraphService<TestServiceData>
|
||||
kTestServiceDisallowDefaultInitialization(
|
||||
"kTestServiceDisallowDefaultInitialization",
|
||||
GraphServiceBase::kDisallowDefaultInitialization);
|
||||
|
||||
static_assert(std::is_trivially_destructible_v<GraphService<TestServiceData>>,
|
||||
"GraphService is not trivially destructible");
|
||||
|
||||
class FailOnUnavailableOptionalDisallowDefaultInitServiceCalculator
|
||||
: public CalculatorBase {
|
||||
|
||||
@@ -20,6 +20,7 @@ def mediapipe_cc_test(
|
||||
open_gl_driver = None,
|
||||
emulator_mini_boot = True,
|
||||
requires_full_emulation = True,
|
||||
android_devices = {},
|
||||
# wasm_web_test arguments
|
||||
browsers = None,
|
||||
**kwargs):
|
||||
|
||||
@@ -94,7 +94,7 @@ const Packet& OutputStreamShard::Header() const {
|
||||
// binary. This function can be defined in the .cc file because only two
|
||||
// versions are ever instantiated, and all call sites are within this .cc file.
|
||||
template <typename T>
|
||||
Status OutputStreamShard::AddPacketInternal(T&& packet) {
|
||||
absl::Status OutputStreamShard::AddPacketInternal(T&& packet) {
|
||||
if (IsClosed()) {
|
||||
return mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "Packet sent to closed stream \"" << Name() << "\".";
|
||||
@@ -113,7 +113,7 @@ Status OutputStreamShard::AddPacketInternal(T&& packet) {
|
||||
<< timestamp.DebugString();
|
||||
}
|
||||
|
||||
Status result = output_stream_spec_->packet_type->Validate(packet);
|
||||
absl::Status result = output_stream_spec_->packet_type->Validate(packet);
|
||||
if (!result.ok()) {
|
||||
return StatusBuilder(result, MEDIAPIPE_LOC).SetPrepend() << absl::StrCat(
|
||||
"Packet type mismatch on calculator outputting to stream \"",
|
||||
@@ -132,14 +132,14 @@ Status OutputStreamShard::AddPacketInternal(T&& packet) {
|
||||
}
|
||||
|
||||
void OutputStreamShard::AddPacket(const Packet& packet) {
|
||||
Status status = AddPacketInternal(packet);
|
||||
absl::Status status = AddPacketInternal(packet);
|
||||
if (!status.ok()) {
|
||||
output_stream_spec_->TriggerErrorCallback(status);
|
||||
}
|
||||
}
|
||||
|
||||
void OutputStreamShard::AddPacket(Packet&& packet) {
|
||||
Status status = AddPacketInternal(std::move(packet));
|
||||
absl::Status status = AddPacketInternal(std::move(packet));
|
||||
if (!status.ok()) {
|
||||
output_stream_spec_->TriggerErrorCallback(status);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Rules implementation for mediapipe_proto_alias.bzl, do not load directly."""
|
||||
|
||||
def _copy_header_impl(ctx):
|
||||
source = ctx.attr.source.replace("//", "").replace(":", "/")
|
||||
files = []
|
||||
for dep in ctx.attr.deps:
|
||||
for header in dep[CcInfo].compilation_context.direct_headers:
|
||||
if (header.short_path == source):
|
||||
files.append(header)
|
||||
if len(files) != 1:
|
||||
fail("Expected exactly 1 source, got ", str(files))
|
||||
dest_file = ctx.actions.declare_file(ctx.attr.filename)
|
||||
|
||||
# Use expand_template() with no substitutions as a simple copier.
|
||||
ctx.actions.expand_template(
|
||||
template = files[0],
|
||||
output = dest_file,
|
||||
substitutions = {},
|
||||
)
|
||||
return [DefaultInfo(files = depset([dest_file]))]
|
||||
|
||||
copy_header = rule(
|
||||
implementation = _copy_header_impl,
|
||||
attrs = {
|
||||
"filename": attr.string(),
|
||||
"source": attr.string(),
|
||||
"deps": attr.label_list(providers = [CcInfo]),
|
||||
},
|
||||
output_to_genfiles = True,
|
||||
outputs = {"out": "%{filename}"},
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user