Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9fbe868e5 | ||
|
|
67bd8a2bf0 | ||
|
|
59ee17c1f3 | ||
|
|
b09cc090d4 | ||
|
|
cd2b69d58c | ||
|
|
5d028d923b | ||
|
|
b133b0f200 | ||
|
|
b6e680647c | ||
|
|
7bad8fce62 | ||
|
|
024f7bf0f1 | ||
|
|
16e5d7242d | ||
|
|
4c68eb4a70 | ||
|
|
a3d36eee32 | ||
|
|
1722d4b8a2 | ||
|
|
3b6d3c4058 | ||
|
|
252a5713c7 | ||
|
|
de4fbc10e6 | ||
|
|
d144e564d8 | ||
|
|
dd02df1dbe | ||
|
|
66b377c825 | ||
|
|
bf5185f122 | ||
|
|
a2823541e6 | ||
|
|
ae6be10afe | ||
|
|
38ee2603a7 | ||
|
|
86b3283b2f | ||
|
|
7d470a1335 | ||
|
|
d16cc3be5b | ||
|
|
137867d088 | ||
|
|
446d7cf6b6 | ||
|
|
90f72bd851 | ||
|
|
4285aeddfc | ||
|
|
37287925b0 | ||
|
|
48bcbb115f | ||
|
|
9437483827 | ||
|
|
d030c13931 | ||
|
|
fce372d153 | ||
|
|
c6fea4c9d9 | ||
|
|
259b48e082 | ||
|
|
a2a63e3876 | ||
|
|
412ab42d1f | ||
|
|
cc1a02c54f | ||
|
|
b27c562e45 | ||
|
|
61bc4556af | ||
|
|
423c21b454 | ||
|
|
785d266e3f | ||
|
|
59a398924f | ||
|
|
dc9216dc59 | ||
|
|
af67642055 | ||
|
|
731d2b9536 | ||
|
|
b83cfcc9b5 | ||
|
|
9d45360bc9 | ||
|
|
f5df228d9b | ||
|
|
71a47bb18b | ||
|
|
c27a7c1e10 | ||
|
|
7d4ec1e24d | ||
|
|
294687295d |
@@ -1,24 +1,42 @@
|
||||
# The bazelrc file for MediaPipe OSS.
|
||||
|
||||
# Tensorflow needs remote repo
|
||||
common --experimental_repo_remote_exec
|
||||
|
||||
# Basic build settings
|
||||
build --jobs 128
|
||||
build --define='absl=1'
|
||||
build --cxxopt='-std=c++11'
|
||||
build --copt='-Wno-sign-compare'
|
||||
build --copt='-Wno-unused-function'
|
||||
build --copt='-Wno-uninitialized'
|
||||
build --copt='-Wno-unused-result'
|
||||
build --copt='-Wno-comment'
|
||||
build --copt='-Wno-return-type'
|
||||
build --copt='-Wno-unused-local-typedefs'
|
||||
build --copt='-Wno-ignored-attributes'
|
||||
# Temporarily set the incompatiblity flag for Bazel 0.27.0 and above
|
||||
build --incompatible_disable_deprecated_attr_params=false
|
||||
build --enable_platform_specific_config
|
||||
|
||||
# Linux
|
||||
build:linux --cxxopt=-std=c++14
|
||||
build:linux --host_cxxopt=-std=c++14
|
||||
build:linux --copt=-w
|
||||
|
||||
# windows
|
||||
build:windows --cxxopt=/std:c++14
|
||||
build:windows --host_cxxopt=/std:c++14
|
||||
build:windows --copt=/w
|
||||
# For using M_* math constants on Windows with MSVC.
|
||||
build:windows --copt=/D_USE_MATH_DEFINES
|
||||
build:windows --host_copt=/D_USE_MATH_DEFINES
|
||||
|
||||
# macOS
|
||||
build:macos --cxxopt=-std=c++14
|
||||
build:macos --host_cxxopt=-std=c++14
|
||||
build:macos --copt=-w
|
||||
|
||||
# Sets the default Apple platform to macOS.
|
||||
build --apple_platform_type=macos
|
||||
|
||||
# Allow debugging with XCODE
|
||||
build --apple_generate_dsym
|
||||
|
||||
# Android configs.
|
||||
# Note: the documentation tells us to use @androidndk//:default_crosstool, but
|
||||
# the automatic configuration transition uses //external:android/crosstool.
|
||||
# Using it here works and spares us from having two different config_settings
|
||||
# for Android.
|
||||
build:android --crosstool_top=//external:android/crosstool
|
||||
build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
|
||||
build:android --linkopt=-landroid
|
||||
@@ -34,3 +52,34 @@ build:android_arm --fat_apk_cpu=armeabi-v7a
|
||||
build:android_arm64 --config=android
|
||||
build:android_arm64 --cpu=arm64-v8a
|
||||
build:android_arm64 --fat_apk_cpu=arm64-v8a
|
||||
|
||||
# iOS configs.
|
||||
build:ios --apple_platform_type=ios
|
||||
|
||||
build:ios_i386 --config=ios
|
||||
build:ios_i386 --cpu=ios_i386
|
||||
build:ios_i386 --watchos_cpus=i386
|
||||
|
||||
build:ios_x86_64 --config=ios
|
||||
build:ios_x86_64 --cpu=ios_x86_64
|
||||
build:ios_x86_64 --watchos_cpus=i386
|
||||
|
||||
build:ios_armv7 --config=ios
|
||||
build:ios_armv7 --cpu=ios_armv7
|
||||
build:ios_armv7 --watchos_cpus=armv7k
|
||||
|
||||
build:ios_arm64 --config=ios
|
||||
build:ios_arm64 --cpu=ios_arm64
|
||||
build:ios_arm64 --watchos_cpus=armv7k
|
||||
|
||||
build:ios_arm64e --config=ios
|
||||
build:ios_arm64e --cpu=ios_arm64e
|
||||
build:ios_arm64e --watchos_cpus=armv7k
|
||||
|
||||
build:ios_fat --config=ios
|
||||
build:ios_fat --ios_multi_cpus=armv7,arm64
|
||||
build:ios_fat --watchos_cpus=armv7k
|
||||
|
||||
build:darwin_x86_64 --apple_platform_type=macos
|
||||
build:darwin_x86_64 --macos_minimum_os=10.12
|
||||
build:darwin_x86_64 --cpu=darwin_x86_64
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
bazel-*
|
||||
mediapipe/MediaPipe.xcodeproj
|
||||
mediapipe/MediaPipe.tulsiproj/*.tulsiconf-user
|
||||
mediapipe/provisioning_profile.mobileprovision
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright 2019 The MediaPipeOSS Authors.
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM ubuntu:latest
|
||||
FROM ubuntu:18.04
|
||||
|
||||
MAINTAINER <[email protected]>
|
||||
|
||||
@@ -24,20 +24,37 @@ ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
ffmpeg \
|
||||
git \
|
||||
wget \
|
||||
unzip \
|
||||
python \
|
||||
python3-dev \
|
||||
python3-opencv \
|
||||
python3-pip \
|
||||
libopencv-core-dev \
|
||||
libopencv-highgui-dev \
|
||||
libopencv-imgproc-dev \
|
||||
libopencv-video-dev \
|
||||
&& \
|
||||
libopencv-calib3d-dev \
|
||||
libopencv-features2d-dev \
|
||||
software-properties-common && \
|
||||
add-apt-repository -y ppa:openjdk-r/ppa && \
|
||||
apt-get update && apt-get install -y openjdk-8-jdk && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip3 install --upgrade setuptools
|
||||
RUN pip3 install wheel
|
||||
RUN pip3 install future
|
||||
RUN pip3 install six==1.14.0
|
||||
RUN pip3 install tensorflow==1.14.0
|
||||
RUN pip3 install tf_slim
|
||||
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# Install bazel
|
||||
ARG BAZEL_VERSION=0.26.1
|
||||
ARG BAZEL_VERSION=2.0.0
|
||||
RUN mkdir /bazel && \
|
||||
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
|
||||
azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||
@@ -49,4 +66,4 @@ azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||
COPY . /mediapipe/
|
||||
|
||||
# If we want the docker image to contain the pre-built object_detection_offline_demo binary, do the following
|
||||
# RUN bazel build -c opt --define 'MEDIAPIPE_DISABLE_GPU=1' mediapipe/examples/desktop/demo:object_detection_tensorflow_demo
|
||||
# RUN bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/demo:object_detection_tensorflow_demo
|
||||
|
||||
@@ -1,37 +1,141 @@
|
||||

|
||||
=======================================================================
|
||||
---
|
||||
layout: default
|
||||
title: Home
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.
|
||||

|
||||
|
||||

|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
## Installation
|
||||
Follow these [instructions](mediapipe/docs/install.md).
|
||||
## Cross-platform ML solutions made simple
|
||||
|
||||
[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
|
||||
and developers to build world-class ML solutions and applications for mobile,
|
||||
desktop/cloud, web and IoT devices.
|
||||
|
||||
 | 
|
||||
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
|
||||
***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
|
||||
 | 
|
||||
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
|
||||
|
||||
## ML solutions in MediaPipe
|
||||
|
||||
Face Detection | Face Mesh | Hands | Hair Segmentation
|
||||
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :---------------:
|
||||
[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/hair_segmentation)
|
||||
|
||||
Object Detection | Box Tracking | Objectron | KNIFT
|
||||
:----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
|
||||
[](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
|
||||
|
||||
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
|
||||
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
|
||||
|
||||
[]() | Android | iOS | Desktop | Web | Coral
|
||||
:---------------------------------------------------------------------------- | :-----: | :-: | :-----: | :-: | :---:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ |
|
||||
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | ✅ |
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | |
|
||||
|
||||
## MediaPipe on the Web
|
||||
|
||||
MediaPipe on the Web is an effort to run the same ML solutions built for mobile
|
||||
and desktop also in web browsers. The official API is under construction, but
|
||||
the core technology has been proven effective. Please see
|
||||
[MediaPipe on the Web](https://developers.googleblog.com/2020/01/mediapipe-on-web.html)
|
||||
in Google Developers Blog for details.
|
||||
|
||||
You can use the following links to load a demo in the MediaPipe Visualizer, and
|
||||
over there click the "Runner" icon in the top bar like shown below. The demos
|
||||
use your webcam video as input, which is processed all locally in real-time and
|
||||
never leaves your device.
|
||||
|
||||

|
||||
|
||||
* [MediaPipe Face Detection](https://viz.mediapipe.dev/demo/face_detection)
|
||||
* [MediaPipe Hands](https://viz.mediapipe.dev/demo/hand_tracking)
|
||||
* [MediaPipe Hands (palm/hand detection only)](https://viz.mediapipe.dev/demo/hand_detection)
|
||||
* [MediaPipe Hair Segmentation](https://viz.mediapipe.dev/demo/hair_segmentation)
|
||||
|
||||
## Getting started
|
||||
See mobile and desktop [examples](mediapipe/docs/examples.md).
|
||||
|
||||
## Documentation
|
||||
[MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev)
|
||||
Learn how to [install](https://google.github.io/mediapipe/getting_started/install)
|
||||
MediaPipe and
|
||||
[build example applications](https://google.github.io/mediapipe/getting_started/building_examples),
|
||||
and start exploring our ready-to-use
|
||||
[solutions](https://google.github.io/mediapipe/solutions/solutions) that you can
|
||||
further extend and customize.
|
||||
|
||||
Check out the [Examples page](https://mediapipe.readthedocs.io/en/latest/examples.html) for tutorials on how to use MediaPipe. [Concepts page](https://mediapipe.readthedocs.io/en/latest/concepts.html) for basic definitions
|
||||
|
||||
## Visualizing MediaPipe graphs
|
||||
A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.dev/). Please also see instructions [here](mediapipe/docs/visualizer.md).
|
||||
|
||||
## Community forum
|
||||
* [discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
|
||||
The source code is hosted in the
|
||||
[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
|
||||
run code search using
|
||||
[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
|
||||
|
||||
## Publications
|
||||
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
||||
|
||||
* [MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
|
||||
in Google Developers Blog
|
||||
* [Alfred Camera: Smart camera features using MediaPipe](https://developers.googleblog.com/2020/03/alfred-camera-smart-camera-features-using-mediapipe.html)
|
||||
in Google Developers Blog
|
||||
* [Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
|
||||
in Google AI Blog
|
||||
* [AutoFlip: An Open Source Framework for Intelligent Video Reframing](https://ai.googleblog.com/2020/02/autoflip-open-source-framework-for.html)
|
||||
in Google AI Blog
|
||||
* [MediaPipe on the Web](https://developers.googleblog.com/2020/01/mediapipe-on-web.html)
|
||||
in Google Developers Blog
|
||||
* [Object Detection and Tracking using MediaPipe](https://developers.googleblog.com/2019/12/object-detection-and-tracking-using-mediapipe.html)
|
||||
in Google Developers Blog
|
||||
* [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
|
||||
in Google AI Blog
|
||||
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
|
||||
|
||||
## Videos
|
||||
|
||||
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
|
||||
|
||||
## Events
|
||||
[Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
|
||||
|
||||
## Alpha Disclaimer
|
||||
MediaPipe is currently in alpha for v0.5. We are still making breaking API changes and expect to get to stable API by v1.0.
|
||||
* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
|
||||
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
|
||||
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
|
||||
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
|
||||
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
|
||||
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
|
||||
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
|
||||
Seoul, Korea ICCV
|
||||
2019](https://research.google.com/youtube8m/workshop2019/index.html)
|
||||
* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
|
||||
* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
|
||||
([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
|
||||
* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
|
||||
|
||||
## Community
|
||||
|
||||
* [Awesome MediaPipe](https://mediapipe.org) - A curated list of awesome
|
||||
MediaPipe related frameworks, libraries and software
|
||||
* [Slack community](https://https://mediapipe.page.link/joinslack) for MediaPipe users
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
|
||||
community discussion around MediaPipe
|
||||
|
||||
## Alpha disclaimer
|
||||
|
||||
MediaPipe is currently in alpha at v0.7. We may be still making breaking API
|
||||
changes and expect to get to stable APIs by v1.0.
|
||||
|
||||
## Contributing
|
||||
We welcome contributions. Please follow these [guidelines](./CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to the MediaPipe Stack Overflow with a 'mediapipe' tag.
|
||||
We welcome contributions. Please follow these
|
||||
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
|
||||
|
||||
We use GitHub issues for tracking requests and bugs. Please post questions to
|
||||
the MediaPipe Stack Overflow with a `mediapipe` tag.
|
||||
|
||||
@@ -2,33 +2,54 @@ workspace(name = "mediapipe")
|
||||
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
skylib_version = "0.9.0"
|
||||
http_archive(
|
||||
name = "bazel_skylib",
|
||||
sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d",
|
||||
strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b",
|
||||
urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"],
|
||||
type = "tar.gz",
|
||||
url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
|
||||
sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
|
||||
)
|
||||
load("@bazel_skylib//lib:versions.bzl", "versions")
|
||||
versions.check(minimum_bazel_version = "0.23.0")
|
||||
versions.check(minimum_bazel_version = "2.0.0")
|
||||
|
||||
# ABSL cpp library.
|
||||
|
||||
# ABSL cpp library lts_2020_02_25
|
||||
http_archive(
|
||||
name = "com_google_absl",
|
||||
# Head commit on 2019-04-12.
|
||||
# TODO: Switch to the latest absl version when the problem gets
|
||||
# fixed.
|
||||
urls = [
|
||||
"https://github.com/abseil/abseil-cpp/archive/a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a.tar.gz",
|
||||
"https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz",
|
||||
],
|
||||
sha256 = "d437920d1434c766d22e85773b899c77c672b8b4865d5dc2cd61a29fdff3cf03",
|
||||
strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
|
||||
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
|
||||
patches = [
|
||||
"@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "abseil-cpp-20200225",
|
||||
sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353"
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "rules_cc",
|
||||
strip_prefix = "rules_cc-master",
|
||||
urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
|
||||
)
|
||||
|
||||
# GoogleTest/GoogleMock framework. Used by most unit-tests.
|
||||
# Last updated 2020-06-30.
|
||||
http_archive(
|
||||
name = "com_google_googletest",
|
||||
urls = ["https://github.com/google/googletest/archive/master.zip"],
|
||||
strip_prefix = "googletest-master",
|
||||
name = "com_google_googletest",
|
||||
urls = ["https://github.com/google/googletest/archive/aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e.zip"],
|
||||
patches = [
|
||||
# fix for https://github.com/google/googletest/issues/2817
|
||||
"@//third_party:com_google_googletest_9d580ea80592189e6d44fa35bcf9cdea8bf620d6.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "googletest-aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e",
|
||||
sha256 = "04a1751f94244307cebe695a69cc945f9387a80b0ef1af21394a490697c5c895",
|
||||
)
|
||||
|
||||
# Google Benchmark library.
|
||||
@@ -42,21 +63,43 @@ http_archive(
|
||||
# gflags needed by glog
|
||||
http_archive(
|
||||
name = "com_github_gflags_gflags",
|
||||
sha256 = "6e16c8bc91b1310a44f3965e616383dbda48f83e8c1eaa2370a215057b00cabe",
|
||||
strip_prefix = "gflags-77592648e3f3be87d6c7123eb81cbad75f9aef5a",
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
|
||||
"https://github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
|
||||
],
|
||||
strip_prefix = "gflags-2.2.2",
|
||||
sha256 = "19713a36c9f32b33df59d1c79b4958434cb005b5b47dc5400a7a4b078111d9b5",
|
||||
url = "https://github.com/gflags/gflags/archive/v2.2.2.zip",
|
||||
)
|
||||
|
||||
# glog
|
||||
# glog v0.3.5
|
||||
# TODO: Migrate MediaPipe to use com_github_glog_glog on all platforms.
|
||||
http_archive(
|
||||
name = "com_google_glog",
|
||||
name = "com_github_glog_glog_v_0_3_5",
|
||||
url = "https://github.com/google/glog/archive/v0.3.5.zip",
|
||||
sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
|
||||
strip_prefix = "glog-0.3.5",
|
||||
build_file = "@//third_party:glog.BUILD",
|
||||
patches = [
|
||||
"@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
)
|
||||
|
||||
# 2020-02-16
|
||||
http_archive(
|
||||
name = "com_github_glog_glog",
|
||||
strip_prefix = "glog-3ba8976592274bc1f907c402ce22558011d6fc5e",
|
||||
sha256 = "feca3c7e29a693cab7887409756d89d342d4a992d54d7c5599bebeae8f7b50be",
|
||||
urls = [
|
||||
"https://github.com/google/glog/archive/3ba8976592274bc1f907c402ce22558011d6fc5e.zip",
|
||||
],
|
||||
)
|
||||
|
||||
# easyexif
|
||||
http_archive(
|
||||
name = "easyexif",
|
||||
url = "https://github.com/mayanklahiri/easyexif/archive/master.zip",
|
||||
strip_prefix = "easyexif-master",
|
||||
build_file = "@//third_party:easyexif.BUILD",
|
||||
)
|
||||
|
||||
# libyuv
|
||||
@@ -66,11 +109,206 @@ http_archive(
|
||||
build_file = "@//third_party:libyuv.BUILD",
|
||||
)
|
||||
|
||||
# Note: protobuf-javalite is no longer released as a separate download, it's included in the main Java download.
|
||||
# ...but the Java download is currently broken, so we use the "source" download.
|
||||
http_archive(
|
||||
name = "com_google_protobuf_javalite",
|
||||
sha256 = "79d102c61e2a479a0b7e5fc167bcfaa4832a0c6aad4a75fa7da0480564931bcc",
|
||||
strip_prefix = "protobuf-384989534b2246d413dbcd750744faab2607b516",
|
||||
urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"],
|
||||
sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
|
||||
strip_prefix = "protobuf-3.11.4",
|
||||
urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "com_google_protobuf",
|
||||
sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
|
||||
strip_prefix = "protobuf-3.11.4",
|
||||
urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
|
||||
patches = [
|
||||
"@//third_party:com_google_protobuf_fixes.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "com_google_audio_tools",
|
||||
strip_prefix = "multichannel-audio-tools-master",
|
||||
urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "ceres_solver",
|
||||
url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
|
||||
patches = [
|
||||
"@//third_party:ceres_solver_compatibility_fixes.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "ceres-solver-1.14.0",
|
||||
sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "linux_ffmpeg",
|
||||
build_file = "@//third_party:ffmpeg_linux.BUILD",
|
||||
path = "/usr"
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "macos_opencv",
|
||||
build_file = "@//third_party:opencv_macos.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "macos_ffmpeg",
|
||||
build_file = "@//third_party:ffmpeg_macos.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "windows_opencv",
|
||||
build_file = "@//third_party:opencv_windows.BUILD",
|
||||
path = "C:\\opencv\\build",
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "android_opencv",
|
||||
build_file = "@//third_party:opencv_android.BUILD",
|
||||
strip_prefix = "OpenCV-android-sdk",
|
||||
type = "zip",
|
||||
url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip",
|
||||
)
|
||||
|
||||
# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
|
||||
# trigger duplicate symbol errors in the linking stage of building a mediapipe ios app.
|
||||
# To get a higher version of OpenCV for iOS, opencv2.framework needs to be built from source with
|
||||
# '-DBUILD_PROTOBUF=OFF -DBUILD_opencv_dnn=OFF'.
|
||||
http_archive(
|
||||
name = "ios_opencv",
|
||||
sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2",
|
||||
build_file = "@//third_party:opencv_ios.BUILD",
|
||||
type = "zip",
|
||||
url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
|
||||
)
|
||||
|
||||
# You may run setup_android.sh to install Android SDK and NDK.
|
||||
android_ndk_repository(
|
||||
name = "androidndk",
|
||||
)
|
||||
|
||||
android_sdk_repository(
|
||||
name = "androidsdk",
|
||||
)
|
||||
|
||||
# iOS basic build deps.
|
||||
|
||||
http_archive(
|
||||
name = "build_bazel_rules_apple",
|
||||
sha256 = "7a7afdd4869bb201c9352eed2daf37294d42b093579b70423490c1b4d4f6ce42",
|
||||
url = "https://github.com/bazelbuild/rules_apple/releases/download/0.19.0/rules_apple.0.19.0.tar.gz",
|
||||
patches = [
|
||||
# Bypass checking ios unit test runner when building MP ios applications.
|
||||
"@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
)
|
||||
|
||||
load(
|
||||
"@build_bazel_rules_apple//apple:repositories.bzl",
|
||||
"apple_rules_dependencies",
|
||||
)
|
||||
|
||||
apple_rules_dependencies()
|
||||
|
||||
load(
|
||||
"@build_bazel_rules_swift//swift:repositories.bzl",
|
||||
"swift_rules_dependencies",
|
||||
)
|
||||
|
||||
swift_rules_dependencies()
|
||||
|
||||
http_archive(
|
||||
name = "build_bazel_apple_support",
|
||||
sha256 = "122ebf7fe7d1c8e938af6aeaee0efe788a3a2449ece5a8d6a428cb18d6f88033",
|
||||
urls = [
|
||||
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
|
||||
"https://github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
|
||||
],
|
||||
)
|
||||
|
||||
load(
|
||||
"@build_bazel_apple_support//lib:repositories.bzl",
|
||||
"apple_support_dependencies",
|
||||
)
|
||||
|
||||
apple_support_dependencies()
|
||||
|
||||
# More iOS deps.
|
||||
|
||||
http_archive(
|
||||
name = "google_toolbox_for_mac",
|
||||
url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip",
|
||||
sha256 = "e3ac053813c989a88703556df4dc4466e424e30d32108433ed6beaec76ba4fdc",
|
||||
strip_prefix = "google-toolbox-for-mac-2.2.1",
|
||||
build_file = "@//third_party:google_toolbox_for_mac.BUILD",
|
||||
)
|
||||
|
||||
# Maven dependencies.
|
||||
|
||||
RULES_JVM_EXTERNAL_TAG = "3.2"
|
||||
RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
|
||||
|
||||
http_archive(
|
||||
name = "rules_jvm_external",
|
||||
strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
|
||||
sha256 = RULES_JVM_EXTERNAL_SHA,
|
||||
url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
|
||||
)
|
||||
|
||||
load("@rules_jvm_external//:defs.bzl", "maven_install")
|
||||
|
||||
# Important: there can only be one maven_install rule. Add new maven deps here.
|
||||
maven_install(
|
||||
name = "maven",
|
||||
artifacts = [
|
||||
"junit:junit:4.12",
|
||||
"androidx.test.espresso:espresso-core:3.1.1",
|
||||
"org.hamcrest:hamcrest-library:1.3",
|
||||
"androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
"androidx.lifecycle:lifecycle-common:2.2.0",
|
||||
"androidx.annotation:annotation:aar:1.1.0",
|
||||
"androidx.appcompat:appcompat:aar:1.1.0-rc01",
|
||||
"androidx.camera:camera-core:aar:1.0.0-alpha06",
|
||||
"androidx.camera:camera-camera2:aar:1.0.0-alpha06",
|
||||
"androidx.constraintlayout:constraintlayout:aar:1.1.3",
|
||||
"androidx.core:core:aar:1.1.0-rc03",
|
||||
"androidx.legacy:legacy-support-v4:aar:1.0.0",
|
||||
"androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
|
||||
"com.google.android.material:material:aar:1.0.0-rc01",
|
||||
"com.google.code.findbugs:jsr305:3.0.2",
|
||||
"com.google.flogger:flogger-system-backend:0.3.1",
|
||||
"com.google.flogger:flogger:0.3.1",
|
||||
"com.google.guava:guava:27.0.1-android",
|
||||
],
|
||||
repositories = [
|
||||
"https://jcenter.bintray.com",
|
||||
"https://maven.google.com",
|
||||
"https://dl.google.com/dl/android/maven2",
|
||||
"https://repo1.maven.org/maven2",
|
||||
],
|
||||
fetch_sources = True,
|
||||
version_conflict_policy = "pinned",
|
||||
)
|
||||
|
||||
# Needed by TensorFlow
|
||||
@@ -84,110 +322,25 @@ http_archive(
|
||||
],
|
||||
)
|
||||
|
||||
# TensorFlow r1.14-rc0
|
||||
#Tensorflow repo should always go after the other external dependencies.
|
||||
# 2020-05-11
|
||||
_TENSORFLOW_GIT_COMMIT = "7c09d15f9fcc14343343c247ebf5b8e0afe3e4aa"
|
||||
_TENSORFLOW_SHA256= "673d00cbd2676ae43df1993e0d28c10b5ffbe96d9e2ab29f88a77b43c0211299"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
strip_prefix = "tensorflow-1.14.0-rc0",
|
||||
sha256 = "76404a6157a45e8d7a07e4f5690275256260130145924c2a7c73f6eda2a3de10",
|
||||
urls = ["https://github.com/tensorflow/tensorflow/archive/v1.14.0-rc0.zip"],
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
],
|
||||
patches = [
|
||||
"@//third_party:org_tensorflow_compatibility_fixes.diff",
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
|
||||
sha256 = _TENSORFLOW_SHA256,
|
||||
)
|
||||
|
||||
load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
|
||||
tf_workspace(tf_repo_name = "org_tensorflow")
|
||||
|
||||
# Please run $ sudo apt-get install libopencv-dev
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
# Please run $ brew install opencv
|
||||
new_local_repository(
|
||||
name = "macos_opencv",
|
||||
build_file = "@//third_party:opencv_macos.BUILD",
|
||||
path = "/usr",
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "android_opencv",
|
||||
sha256="056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b",
|
||||
build_file = "@//third_party:opencv_android.BUILD",
|
||||
strip_prefix = "OpenCV-android-sdk",
|
||||
type = "zip",
|
||||
url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip",
|
||||
)
|
||||
|
||||
RULES_JVM_EXTERNAL_TAG = "2.2"
|
||||
RULES_JVM_EXTERNAL_SHA = "f1203ce04e232ab6fdd81897cf0ff76f2c04c0741424d192f28e65ae752ce2d6"
|
||||
|
||||
http_archive(
|
||||
name = "rules_jvm_external",
|
||||
strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
|
||||
sha256 = RULES_JVM_EXTERNAL_SHA,
|
||||
url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
|
||||
)
|
||||
|
||||
load("@rules_jvm_external//:defs.bzl", "maven_install")
|
||||
|
||||
maven_install(
|
||||
artifacts = [
|
||||
"com.android.support.constraint:constraint-layout:aar:1.0.2",
|
||||
"androidx.appcompat:appcompat:aar:1.0.2",
|
||||
],
|
||||
repositories = [
|
||||
"https://dl.google.com/dl/android/maven2",
|
||||
],
|
||||
)
|
||||
|
||||
maven_server(
|
||||
name = "google_server",
|
||||
url = "https://dl.google.com/dl/android/maven2",
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
name = "androidx_lifecycle",
|
||||
artifact = "androidx.lifecycle:lifecycle-common:2.0.0",
|
||||
sha1 = "e070ffae07452331bc5684734fce6831d531785c",
|
||||
server = "google_server",
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
name = "androidx_concurrent_futures",
|
||||
artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
|
||||
server = "google_server",
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
name = "com_google_guava_android",
|
||||
artifact = "com.google.guava:guava:27.0.1-android",
|
||||
sha1 = "b7e1c37f66ef193796ccd7ea6e80c2b05426182d",
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
name = "com_google_common_flogger",
|
||||
artifact = "com.google.flogger:flogger:0.3.1",
|
||||
sha1 = "585030fe1ec709760cbef997a459729fb965df0e",
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
name = "com_google_common_flogger_system_backend",
|
||||
artifact = "com.google.flogger:flogger-system-backend:0.3.1",
|
||||
sha1 = "287b569d76abcd82f9de87fe41829fbc7ebd8ac9",
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
name = "com_google_code_findbugs",
|
||||
artifact = "com.google.code.findbugs:jsr305:3.0.2",
|
||||
sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d",
|
||||
)
|
||||
|
||||
# You may run setup_android.sh to install Android SDK and NDK.
|
||||
android_ndk_repository(
|
||||
name = "androidndk",
|
||||
)
|
||||
|
||||
android_sdk_repository(
|
||||
name = "androidsdk",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# =========================================================================
|
||||
#
|
||||
# Script to build all MediaPipe Android example apps.
|
||||
#
|
||||
# To build all apps and store them in out_dir, and install them:
|
||||
# $ ./build_android_examples.sh -d out_dir
|
||||
# Omitting -d and the associated directory saves all generated APKs in the
|
||||
# current directory.
|
||||
# $ ./build_android_examples.sh -d out_dir --nostrip
|
||||
# Same as above except that the symnbols are not stripped.
|
||||
#
|
||||
# To install the apps already stored in out_dir (after building them with the
|
||||
# usages above):
|
||||
# $ ./build_android_examples.sh -d out_dir -i
|
||||
# Omitting -d and the associated directory assumes the apps are in the
|
||||
# current directory.
|
||||
|
||||
set -e
|
||||
|
||||
function switch_to_opencv_3() {
|
||||
echo "Switching to OpenCV 3"
|
||||
sed -i -e 's:4.0.1/opencv-4.0.1:3.4.3/opencv-3.4.3:g' WORKSPACE
|
||||
sed -i -e 's:libopencv_java4:libopencv_java3:g' third_party/opencv_android.BUILD
|
||||
}
|
||||
|
||||
function switch_to_opencv_4() {
|
||||
echo "Switching to OpenCV 4"
|
||||
sed -i -e 's:3.4.3/opencv-3.4.3:4.0.1/opencv-4.0.1:g' WORKSPACE
|
||||
sed -i -e 's:libopencv_java3:libopencv_java4:g' third_party/opencv_android.BUILD
|
||||
}
|
||||
|
||||
out_dir="."
|
||||
strip=true
|
||||
install_only=false
|
||||
app_dir="mediapipe/examples/android/src/java/com/google/mediapipe/apps"
|
||||
bin_dir="bazel-bin"
|
||||
declare -a default_bazel_flags=(build -c opt --config=android_arm64)
|
||||
|
||||
while [[ -n $1 ]]; do
|
||||
case $1 in
|
||||
-d)
|
||||
shift
|
||||
out_dir=$1
|
||||
;;
|
||||
--nostrip)
|
||||
strip=false
|
||||
;;
|
||||
-i)
|
||||
install_only=true
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported input argument $1."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
echo "app_dir: $app_dir"
|
||||
echo "out_dir: $out_dir"
|
||||
echo "strip: $strip"
|
||||
|
||||
declare -a apks=()
|
||||
declare -a bazel_flags
|
||||
switch_to_opencv_3
|
||||
|
||||
apps="${app_dir}/*"
|
||||
for app in ${apps}; do
|
||||
if [[ -d "${app}" ]]; then
|
||||
app_name=${app##*/}
|
||||
if [[ ${app_name} == "basic" ]]; then
|
||||
target_name="helloworld"
|
||||
else
|
||||
target_name=${app_name}
|
||||
fi
|
||||
target="${app}:${target_name}"
|
||||
bin="${bin_dir}/${app}/${target_name}.apk"
|
||||
apk="${out_dir}/${target_name}.apk"
|
||||
|
||||
echo "=== Target: ${target}"
|
||||
|
||||
if [[ $install_only == false ]]; then
|
||||
bazel_flags=("${default_bazel_flags[@]}")
|
||||
bazel_flags+=(${target})
|
||||
if [[ $strip == true ]]; then
|
||||
bazel_flags+=(--linkopt=-s)
|
||||
fi
|
||||
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_4
|
||||
fi
|
||||
bazel "${bazel_flags[@]}"
|
||||
cp -f "${bin}" "${apk}"
|
||||
if [[ ${app_name} == "templatematchingcpu" ]]; then
|
||||
switch_to_opencv_3
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${app_name} == "objectdetection3d" ]]; then
|
||||
orig_apk=${apk}
|
||||
apk="${out_dir}/${target_name}_shoes.apk"
|
||||
cp -f "${orig_apk}" "${apk}"
|
||||
apks+=(${apk})
|
||||
|
||||
apk="${out_dir}/${target_name}_chairs.apk"
|
||||
if [[ $install_only == false ]]; then
|
||||
bazel_flags+=(--define chair=true)
|
||||
bazel "${bazel_flags[@]}"
|
||||
cp -f "${bin}" "${apk}"
|
||||
fi
|
||||
fi
|
||||
|
||||
apks+=(${apk})
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Connect your device via adb to install the apps."
|
||||
read -p "Press 'a' to abort, or press any other key to continue ..." -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Aa]$ ]]; then
|
||||
for apk in "${apks[@]}"; do
|
||||
echo "=== Installing $apk"
|
||||
adb install -r "${apk}"
|
||||
done
|
||||
fi
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# =========================================================================
|
||||
#
|
||||
# Script to build all MediaPipe iOS example apps.
|
||||
#
|
||||
# To build all apps and store them in out_dir:
|
||||
# $ ./build_ios_examples.sh -d out_dir
|
||||
# Omitting -d and the associated directory saves all generated IPAs in the
|
||||
# current directory.
|
||||
# $ ./build_ios_examples.sh -d out_dir --nostrip
|
||||
# Same as above except that the symnbols are not stripped.
|
||||
|
||||
set -e
|
||||
|
||||
out_dir="."
|
||||
strip=true
|
||||
app_dir="mediapipe/examples/ios"
|
||||
bin_dir="bazel-bin"
|
||||
declare -a default_bazel_flags=(build -c opt --config=ios_arm64)
|
||||
|
||||
while [[ -n $1 ]]; do
|
||||
case $1 in
|
||||
-d)
|
||||
shift
|
||||
out_dir=$1
|
||||
;;
|
||||
--nostrip)
|
||||
strip=false
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported input argument $1."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
echo "app_dir: $app_dir"
|
||||
echo "out_dir: $out_dir"
|
||||
echo "strip: $strip"
|
||||
|
||||
declare -a bazel_flags
|
||||
|
||||
apps="${app_dir}/*"
|
||||
for app in ${apps}; do
|
||||
if [[ -d "${app}" ]]; then
|
||||
target_name=${app##*/}
|
||||
target="${app}:${target_name}"
|
||||
|
||||
echo "=== Target: ${target}"
|
||||
|
||||
bazel_flags=("${default_bazel_flags[@]}")
|
||||
bazel_flags+=(${target})
|
||||
if [[ $strip == true ]]; then
|
||||
bazel_flags+=(--linkopt=-s)
|
||||
fi
|
||||
|
||||
bazel "${bazel_flags[@]}"
|
||||
cp -f "${bin_dir}/${app}/"*".ipa" "${out_dir}"
|
||||
fi
|
||||
done
|
||||
@@ -0,0 +1,29 @@
|
||||
# Configuration for GitHub Pages
|
||||
|
||||
remote_theme: pmarsceill/just-the-docs
|
||||
|
||||
# Set a path/url to a logo that will be displayed instead of the title
|
||||
logo: "images/logo_horizontal_color.png"
|
||||
|
||||
# Enable or disable the site search
|
||||
search_enabled: true
|
||||
|
||||
# Set the search token separator for hyphenated-word search:
|
||||
search_tokenizer_separator: /[\s/]+/
|
||||
|
||||
# Enable or disable heading anchors
|
||||
heading_anchors: true
|
||||
|
||||
# Aux links for the upper right navigation
|
||||
aux_links:
|
||||
"MediaPipe on GitHub":
|
||||
- "//github.com/google/mediapipe"
|
||||
|
||||
# Footer content appears at the bottom of every page's main content
|
||||
footer_content: "© 2020 GOOGLE LLC | <a href=\"https://policies.google.com/privacy\">PRIVACY POLICY</a> | <a href=\"https://policies.google.com/terms\">TERMS OF SERVICE</a>"
|
||||
|
||||
# Color scheme currently only supports "dark" or nil (default)
|
||||
color_scheme: nil
|
||||
|
||||
# Google Analytics Tracking (optional)
|
||||
ga_tracking: UA-140696581-2
|
||||
@@ -19,7 +19,7 @@ project = 'MediaPipe'
|
||||
author = 'Google LLC'
|
||||
|
||||
# The full version, including alpha/beta/rc tags
|
||||
release = 'v0.5'
|
||||
release = 'v0.7.5'
|
||||
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
@@ -0,0 +1,412 @@
|
||||
---
|
||||
layout: default
|
||||
title: Calculators
|
||||
parent: Framework Concepts
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
# Calculators
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
Each calculator is a node of a graph. We describe how to create a new
|
||||
calculator, how to initialize a calculator, how to perform its calculations,
|
||||
input and output streams, timestamps, and options. Each node in the graph is
|
||||
implemented as a `Calculator`. The bulk of graph execution happens inside its
|
||||
calculators. A calculator may receive zero or more input streams and/or side
|
||||
packets and produces zero or more output streams and/or side packets.
|
||||
|
||||
## CalculatorBase
|
||||
|
||||
A calculator is created by defining a new sub-class of the
|
||||
[`CalculatorBase`](https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.cc)
|
||||
class, implementing a number of methods, and registering the new sub-class with
|
||||
Mediapipe. At a minimum, a new calculator must implement the below four methods
|
||||
|
||||
* `GetContract()`
|
||||
* Calculator authors can specify the expected types of inputs and outputs
|
||||
of a calculator in GetContract(). When a graph is initialized, the
|
||||
framework calls a static method to verify if the packet types of the
|
||||
connected inputs and outputs match the information in this
|
||||
specification.
|
||||
* `Open()`
|
||||
* After a graph starts, the framework calls `Open()`. The input side
|
||||
packets are available to the calculator at this point. `Open()`
|
||||
interprets the node configuration operations (see [Graphs](graphs.md))
|
||||
and prepares the calculator's per-graph-run state. This function may
|
||||
also write packets to calculator outputs. An error during `Open()` can
|
||||
terminate the graph run.
|
||||
* `Process()`
|
||||
* For a calculator with inputs, the framework calls `Process()` repeatedly
|
||||
whenever at least one input stream has a packet available. The framework
|
||||
by default guarantees that all inputs have the same timestamp (see
|
||||
[Synchronization](synchronization.md) for more information). Multiple
|
||||
`Process()` calls can be invoked simultaneously when parallel execution
|
||||
is enabled. If an error occurs during `Process()`, the framework calls
|
||||
`Close()` and the graph run terminates.
|
||||
* `Close()`
|
||||
* After all calls to `Process()` finish or when all input streams close,
|
||||
the framework calls `Close()`. This function is always called if
|
||||
`Open()` was called and succeeded and even if the graph run terminated
|
||||
because of an error. No inputs are available via any input streams
|
||||
during `Close()`, but it still has access to input side packets and
|
||||
therefore may write outputs. After `Close()` returns, the calculator
|
||||
should be considered a dead node. The calculator object is destroyed as
|
||||
soon as the graph finishes running.
|
||||
|
||||
The following are code snippets from
|
||||
[CalculatorBase.h](https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.h).
|
||||
|
||||
```c++
|
||||
class CalculatorBase {
|
||||
public:
|
||||
...
|
||||
|
||||
// The subclasses of CalculatorBase must implement GetContract.
|
||||
// ...
|
||||
static ::MediaPipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
// Open is called before any Process() calls, on a freshly constructed
|
||||
// calculator. Subclasses may override this method to perform necessary
|
||||
// setup, and possibly output Packets and/or set output streams' headers.
|
||||
// ...
|
||||
virtual ::MediaPipe::Status Open(CalculatorContext* cc) {
|
||||
return ::MediaPipe::OkStatus();
|
||||
}
|
||||
|
||||
// Processes the incoming inputs. May call the methods on cc to access
|
||||
// inputs and produce outputs.
|
||||
// ...
|
||||
virtual ::MediaPipe::Status Process(CalculatorContext* cc) = 0;
|
||||
|
||||
// Is called if Open() was called and succeeded. Is called either
|
||||
// immediately after processing is complete or after a graph run has ended
|
||||
// (if an error occurred in the graph). ...
|
||||
virtual ::MediaPipe::Status Close(CalculatorContext* cc) {
|
||||
return ::MediaPipe::OkStatus();
|
||||
}
|
||||
|
||||
...
|
||||
};
|
||||
```
|
||||
|
||||
## Life of a calculator
|
||||
|
||||
During initialization of a MediaPipe graph, the framework calls a
|
||||
`GetContract()` static method to determine what kinds of packets are expected.
|
||||
|
||||
The framework constructs and destroys the entire calculator for each graph run
|
||||
(e.g. once per video or once per image). Expensive or large objects that remain
|
||||
constant across graph runs should be supplied as input side packets so the
|
||||
calculations are not repeated on subsequent runs.
|
||||
|
||||
After initialization, for each run of the graph, the following sequence occurs:
|
||||
|
||||
* `Open()`
|
||||
* `Process()` (repeatedly)
|
||||
* `Close()`
|
||||
|
||||
The framework calls `Open()` to initialize the calculator. `Open()` should
|
||||
interpret any options and set up the calculator's per-graph-run state. `Open()`
|
||||
may obtain input side packets and write packets to calculator outputs. If
|
||||
appropriate, it should call `SetOffset()` to reduce potential packet buffering
|
||||
of input streams.
|
||||
|
||||
If an error occurs during `Open()` or `Process()` (as indicated by one of them
|
||||
returning a non-`Ok` status), the graph run is terminated with no further calls
|
||||
to the calculator's methods, and the calculator is destroyed.
|
||||
|
||||
For a calculator with inputs, the framework calls `Process()` whenever at least
|
||||
one input has a packet available. The framework guarantees that inputs all have
|
||||
the same timestamp, that timestamps increase with each call to `Process()` and
|
||||
that all packets are delivered. As a consequence, some inputs may not have any
|
||||
packets when `Process()` is called. An input whose packet is missing appears to
|
||||
produce an empty packet (with no timestamp).
|
||||
|
||||
The framework calls `Close()` after all calls to `Process()`. All inputs will
|
||||
have been exhausted, but `Close()` has access to input side packets and may
|
||||
write outputs. After Close returns, the calculator is destroyed.
|
||||
|
||||
Calculators with no inputs are referred to as sources. A source calculator
|
||||
continues to have `Process()` called as long as it returns an `Ok` status. A
|
||||
source calculator indicates that it is exhausted by returning a stop status
|
||||
(i.e. MediaPipe::tool::StatusStop).
|
||||
|
||||
## Identifying inputs and outputs
|
||||
|
||||
The public interface to a calculator consists of a set of input streams and
|
||||
output streams. In a CalculatorGraphConfiguration, the outputs from some
|
||||
calculators are connected to the inputs of other calculators using named
|
||||
streams. Stream names are normally lowercase, while input and output tags are
|
||||
normally UPPERCASE. In the example below, the output with tag name `VIDEO` is
|
||||
connected to the input with tag name `VIDEO_IN` using the stream named
|
||||
`video_stream`.
|
||||
|
||||
```proto
|
||||
# Graph describing calculator SomeAudioVideoCalculator
|
||||
node {
|
||||
calculator: "SomeAudioVideoCalculator"
|
||||
input_stream: "INPUT:combined_input"
|
||||
output_stream: "VIDEO:video_stream"
|
||||
}
|
||||
node {
|
||||
calculator: "SomeVideoCalculator"
|
||||
input_stream: "VIDEO_IN:video_stream"
|
||||
output_stream: "VIDEO_OUT:processed_video"
|
||||
}
|
||||
```
|
||||
|
||||
Input and output streams can be identified by index number, by tag name, or by a
|
||||
combination of tag name and index number. You can see some examples of input and
|
||||
output identifiers in the example below. `SomeAudioVideoCalculator` identifies
|
||||
its video output by tag and its audio outputs by the combination of tag and
|
||||
index. The input with tag `VIDEO` is connected to the stream named
|
||||
`video_stream`. The outputs with tag `AUDIO` and indices `0` and `1` are
|
||||
connected to the streams named `audio_left` and `audio_right`.
|
||||
`SomeAudioCalculator` identifies its audio inputs by index only (no tag needed).
|
||||
|
||||
```proto
|
||||
# Graph describing calculator SomeAudioVideoCalculator
|
||||
node {
|
||||
calculator: "SomeAudioVideoCalculator"
|
||||
input_stream: "combined_input"
|
||||
output_stream: "VIDEO:video_stream"
|
||||
output_stream: "AUDIO:0:audio_left"
|
||||
output_stream: "AUDIO:1:audio_right"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "SomeAudioCalculator"
|
||||
input_stream: "audio_left"
|
||||
input_stream: "audio_right"
|
||||
output_stream: "audio_energy"
|
||||
}
|
||||
```
|
||||
|
||||
In the calculator implementation, inputs and outputs are also identified by tag
|
||||
name and index number. In the function below input are output are identified:
|
||||
|
||||
* By index number: The combined input stream is identified simply by index
|
||||
`0`.
|
||||
* By tag name: The video output stream is identified by tag name "VIDEO".
|
||||
* By tag name and index number: The output audio streams are identified by the
|
||||
combination of the tag name `AUDIO` and the index numbers `0` and `1`.
|
||||
|
||||
```c++
|
||||
// c++ Code snippet describing the SomeAudioVideoCalculator GetContract() method
|
||||
class SomeAudioVideoCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
// SetAny() is used to specify that whatever the type of the
|
||||
// stream is, it's acceptable. This does not mean that any
|
||||
// packet is acceptable. Packets in the stream still have a
|
||||
// particular type. SetAny() has the same effect as explicitly
|
||||
// setting the type to be the stream's type.
|
||||
cc->Outputs().Tag("VIDEO").Set<ImageFrame>();
|
||||
cc->Outputs().Get("AUDIO", 0).Set<Matrix>;
|
||||
cc->Outputs().Get("AUDIO", 1).Set<Matrix>;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
```
|
||||
|
||||
## Processing
|
||||
|
||||
`Process()` called on a non-source node must return `::mediapipe::OkStatus()` to
|
||||
indicate that all went well, or any other status code to signal an error
|
||||
|
||||
If a non-source calculator returns `tool::StatusStop()`, then this signals the
|
||||
graph is being cancelled early. In this case, all source calculators and graph
|
||||
input streams will be closed (and remaining Packets will propagate through the
|
||||
graph).
|
||||
|
||||
A source node in a graph will continue to have `Process()` called on it as long
|
||||
as it returns `::mediapipe::OkStatus(`). To indicate that there is no more data
|
||||
to be generated return `tool::StatusStop()`. Any other status indicates an error
|
||||
has occurred.
|
||||
|
||||
`Close()` returns `::mediapipe::OkStatus()` to indicate success. Any other
|
||||
status indicates a failure.
|
||||
|
||||
Here is the basic `Process()` function. It uses the `Input()` method (which can
|
||||
be used only if the calculator has a single input) to request its input data. It
|
||||
then uses `std::unique_ptr` to allocate the memory needed for the output packet,
|
||||
and does the calculations. When done it releases the pointer when adding it to
|
||||
the output stream.
|
||||
|
||||
```c++
|
||||
::util::Status MyCalculator::Process() {
|
||||
const Matrix& input = Input()->Get<Matrix>();
|
||||
std::unique_ptr<Matrix> output(new Matrix(input.rows(), input.cols()));
|
||||
// do your magic here....
|
||||
// output->row(n) = ...
|
||||
Output()->Add(output.release(), InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
```
|
||||
|
||||
## Example calculator
|
||||
|
||||
This section discusses the implementation of `PacketClonerCalculator`, which
|
||||
does a relatively simple job, and is used in many calculator graphs.
|
||||
`PacketClonerCalculator` simply produces a copy of its most recent input
|
||||
packets on demand.
|
||||
|
||||
`PacketClonerCalculator` is useful when the timestamps of arriving data packets
|
||||
are not aligned perfectly. Suppose we have a room with a microphone, light
|
||||
sensor and a video camera that is collecting sensory data. Each of the sensors
|
||||
operates independently and collects data intermittently. Suppose that the output
|
||||
of each sensor is:
|
||||
|
||||
* microphone = loudness in decibels of sound in the room (Integer)
|
||||
* light sensor = brightness of room (Integer)
|
||||
* video camera = RGB image frame of room (ImageFrame)
|
||||
|
||||
Our simple perception pipeline is designed to process sensory data from these 3
|
||||
sensors such that at any time when we have image frame data from the camera that
|
||||
is synchronized with the last collected microphone loudness data and light
|
||||
sensor brightness data. To do this with MediaPipe, our perception pipeline has 3
|
||||
input streams:
|
||||
|
||||
* room_mic_signal - Each packet of data in this input stream is integer data
|
||||
representing how loud audio is in a room with timestamp.
|
||||
* room_lightening_sensor - Each packet of data in this input stream is integer
|
||||
data representing how bright is the room illuminated with timestamp.
|
||||
* room_video_tick_signal - Each packet of data in this input stream is
|
||||
imageframe of video data representing video collected from camera in the
|
||||
room with timestamp.
|
||||
|
||||
Below is the implementation of the `PacketClonerCalculator`. You can see
|
||||
the `GetContract()`, `Open()`, and `Process()` methods as well as the instance
|
||||
variable `current_` which holds the most recent input packets.
|
||||
|
||||
```c++
|
||||
// This takes packets from N+1 streams, A_1, A_2, ..., A_N, B.
|
||||
// For every packet that appears in B, outputs the most recent packet from each
|
||||
// of the A_i on a separate stream.
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// For every packet received on the last stream, output the latest packet
|
||||
// obtained on all other streams. Therefore, if the last stream outputs at a
|
||||
// higher rate than the others, this effectively clones the packets from the
|
||||
// other streams to match the last.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "PacketClonerCalculator"
|
||||
// input_stream: "first_base_signal"
|
||||
// input_stream: "second_base_signal"
|
||||
// input_stream: "tick_signal"
|
||||
// output_stream: "cloned_first_base_signal"
|
||||
// output_stream: "cloned_second_base_signal"
|
||||
// }
|
||||
//
|
||||
class PacketClonerCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
const int tick_signal_index = cc->Inputs().NumEntries() - 1;
|
||||
// cc->Inputs().NumEntries() returns the number of input streams
|
||||
// for the PacketClonerCalculator
|
||||
for (int i = 0; i < tick_signal_index; ++i) {
|
||||
cc->Inputs().Index(i).SetAny();
|
||||
// cc->Inputs().Index(i) returns the input stream pointer by index
|
||||
cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
|
||||
}
|
||||
cc->Inputs().Index(tick_signal_index).SetAny();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) final {
|
||||
tick_signal_index_ = cc->Inputs().NumEntries() - 1;
|
||||
current_.resize(tick_signal_index_);
|
||||
// Pass along the header for each stream if present.
|
||||
for (int i = 0; i < tick_signal_index_; ++i) {
|
||||
if (!cc->Inputs().Index(i).Header().IsEmpty()) {
|
||||
cc->Outputs().Index(i).SetHeader(cc->Inputs().Index(i).Header());
|
||||
// Sets the output stream of index i header to be the same as
|
||||
// the header for the input stream of index i
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) final {
|
||||
// Store input signals.
|
||||
for (int i = 0; i < tick_signal_index_; ++i) {
|
||||
if (!cc->Inputs().Index(i).Value().IsEmpty()) {
|
||||
current_[i] = cc->Inputs().Index(i).Value();
|
||||
}
|
||||
}
|
||||
|
||||
// Output if the tick signal is non-empty.
|
||||
if (!cc->Inputs().Index(tick_signal_index_).Value().IsEmpty()) {
|
||||
for (int i = 0; i < tick_signal_index_; ++i) {
|
||||
if (!current_[i].IsEmpty()) {
|
||||
cc->Outputs().Index(i).AddPacket(
|
||||
current_[i].At(cc->InputTimestamp()));
|
||||
// Add a packet to output stream of index i a packet from inputstream i
|
||||
// with timestamp common to all present inputs
|
||||
//
|
||||
} else {
|
||||
cc->Outputs().Index(i).SetNextTimestampBound(
|
||||
cc->InputTimestamp().NextAllowedInStream());
|
||||
// if current_[i], 1 packet buffer for input stream i is empty, we will set
|
||||
// next allowed timestamp for input stream i to be current timestamp + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<Packet> current_;
|
||||
int tick_signal_index_;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(PacketClonerCalculator);
|
||||
} // namespace mediapipe
|
||||
```
|
||||
|
||||
Typically, a calculator has only a .cc file. No .h is required, because
|
||||
mediapipe uses registration to make calculators known to it. After you have
|
||||
defined your calculator class, register it with a macro invocation
|
||||
REGISTER_CALCULATOR(calculator_class_name).
|
||||
|
||||
Below is a trivial MediaPipe graph that has 3 input streams, 1 node
|
||||
(PacketClonerCalculator) and 3 output streams.
|
||||
|
||||
```proto
|
||||
input_stream: "room_mic_signal"
|
||||
input_stream: "room_lighting_sensor"
|
||||
input_stream: "room_video_tick_signal"
|
||||
|
||||
node {
|
||||
calculator: "PacketClonerCalculator"
|
||||
input_stream: "room_mic_signal"
|
||||
input_stream: "room_lighting_sensor"
|
||||
input_stream: "room_video_tick_signal"
|
||||
output_stream: "cloned_room_mic_signal"
|
||||
output_stream: "cloned_lighting_sensor"
|
||||
}
|
||||
```
|
||||
|
||||
The diagram below shows how the `PacketClonerCalculator` defines its output
|
||||
packets based on its series of input packets.
|
||||
|
||||
|  :
|
||||
| :--------------------------------------------------------------------------: |
|
||||
| *Each time it receives a packet on its TICK input stream, the |
|
||||
: PacketClonerCalculator outputs the most recent packet from each of its input :
|
||||
: streams. The sequence of output packets is determined by the sequene of :
|
||||
: input packets and their timestamps. The timestamps are shows along the right :
|
||||
: side of the diagram.* :
|
||||
@@ -1,24 +1,42 @@
|
||||
# MediaPipe Concepts
|
||||
---
|
||||
layout: default
|
||||
title: Framework Concepts
|
||||
nav_order: 5
|
||||
has_children: true
|
||||
has_toc: false
|
||||
---
|
||||
|
||||
# Framework Concepts
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## The basics
|
||||
|
||||
### Packet
|
||||
|
||||
The basic data flow unit. A packet consists of a numeric timestamp and a shared pointer to an **immutable** payload. The payload can be of any C++ type, and the payload's type is also referred to as the type of the packet. Packets are value classes and can be copied cheaply. Each copy shares ownership of the payload, with reference-counting semantics. Each copy has its own timestamp. [Details](packets.md).
|
||||
The basic data flow unit. A packet consists of a numeric timestamp and a shared
|
||||
pointer to an **immutable** payload. The payload can be of any C++ type, and the
|
||||
payload's type is also referred to as the type of the packet. Packets are value
|
||||
classes and can be copied cheaply. Each copy shares ownership of the payload,
|
||||
with reference-counting semantics. Each copy has its own timestamp. See also
|
||||
[Packet](packets.md).
|
||||
|
||||
### Graph
|
||||
|
||||
MediaPipe processing takes place inside a graph, which defines packet flow paths
|
||||
between **nodes**. A graph can have any number of inputs and outputs, and data
|
||||
flow can branch and merge. Generally data flows forward, but
|
||||
[backward loops](cycles.md) are possible.
|
||||
flow can branch and merge. Generally data flows forward, but backward loops are
|
||||
possible. See [Graphs](graphs.md) for details.
|
||||
|
||||
### Nodes
|
||||
|
||||
Nodes produce and/or consume packets, and they are where the bulk of the graph’s
|
||||
work takes place. They are also known as “calculators”, for historical reasons.
|
||||
Each node’s interface defines a number of input and output **ports**, identified by
|
||||
a tag and/or an index.
|
||||
Each node’s interface defines a number of input and output **ports**, identified
|
||||
by a tag and/or an index. See [Calculators](calculators.md) for details.
|
||||
|
||||
### Streams
|
||||
|
||||
@@ -34,21 +52,23 @@ whereas a stream represents a flow of data that changes over time.
|
||||
### Packet Ports
|
||||
|
||||
A port has an associated type; packets transiting through the port must be of
|
||||
that type. An output stream port can be connected to any number of
|
||||
input stream ports of the same type; each consumer receives a separate copy of
|
||||
the output packets, and has its own queue, so it can consume them at its own
|
||||
pace. Similarly, a side packet output port can be connected to as many side
|
||||
packet input ports as desired.
|
||||
that type. An output stream port can be connected to any number of input stream
|
||||
ports of the same type; each consumer receives a separate copy of the output
|
||||
packets, and has its own queue, so it can consume them at its own pace.
|
||||
Similarly, a side packet output port can be connected to as many side packet
|
||||
input ports as desired.
|
||||
|
||||
A port can be required, meaning that a connection must be made for the graph to
|
||||
be valid, or optional, meaning it may remain unconnected.
|
||||
|
||||
Note: even if a stream connection is required, the stream may not carry a packet for all timestamps.
|
||||
Note: even if a stream connection is required, the stream may not carry a packet
|
||||
for all timestamps.
|
||||
|
||||
## Input and output
|
||||
|
||||
Data flow can originate from **source nodes**, which have no input streams and
|
||||
produce packets spontaneously (e.g. by reading from a file); or from **graph input streams**, which let an application feed packets into a graph.
|
||||
produce packets spontaneously (e.g. by reading from a file); or from **graph
|
||||
input streams**, which let an application feed packets into a graph.
|
||||
|
||||
Similarly, there are **sink nodes** that receive data and write it to various
|
||||
destinations (e.g. a file, a memory buffer, etc.), and an application can also
|
||||
@@ -78,15 +98,15 @@ processed data.
|
||||
|
||||
### Input policies
|
||||
|
||||
The default input policy is deterministic collation of packets by timestamp. A node receives
|
||||
all inputs for the same timestamp at the same time, in an invocation of its
|
||||
Process method; and successive input sets are received in their timestamp order. This can
|
||||
require delaying the processing of some packets until a packet with the same
|
||||
timestamp is received on all input streams, or until it can be guaranteed that a
|
||||
packet with that timestamp will not be arriving on the streams that have not
|
||||
received it.
|
||||
The default input policy is deterministic collation of packets by timestamp. A
|
||||
node receives all inputs for the same timestamp at the same time, in an
|
||||
invocation of its Process method; and successive input sets are received in
|
||||
their timestamp order. This can require delaying the processing of some packets
|
||||
until a packet with the same timestamp is received on all input streams, or
|
||||
until it can be guaranteed that a packet with that timestamp will not be
|
||||
arriving on the streams that have not received it.
|
||||
|
||||
Other policies are also available, implemented using a separate kind of
|
||||
component known as an InputStreamHandler.
|
||||
|
||||
See [scheduling](scheduling_sync.md) for more details.
|
||||
See [Synchronization](synchronization.md) for more details.
|
||||
@@ -1,12 +1,19 @@
|
||||
## Running on GPUs
|
||||
---
|
||||
layout: default
|
||||
title: GPU
|
||||
parent: Framework Concepts
|
||||
nav_order: 5
|
||||
---
|
||||
|
||||
- [Overview](#overview)
|
||||
- [OpenGL Support](#graphconfig)
|
||||
- [Life of a GPU calculator](#life-of-a-gpu-calculator)
|
||||
- [GpuBuffer to ImageFrame converters](#gpubuffer-to-imageframe-converters)
|
||||
# GPU
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
### Overview
|
||||
MediaPipe supports calculator nodes for GPU compute and rendering, and allows combining multiple GPU nodes, as well as mixing them with CPU based calculator nodes. There exist several GPU APIs on mobile platforms (eg, OpenGL ES, Metal and Vulkan). MediaPipe does not attempt to offer a single cross-API GPU abstraction. Individual nodes can be written using different APIs, allowing them to take advantage of platform specific features when needed.
|
||||
|
||||
GPU support is essential for good performance on mobile platforms, especially for real-time video. MediaPipe enables developers to write GPU compatible calculators that support the use of GPU for:
|
||||
@@ -22,14 +29,32 @@ Below are the design principles for GPU support in MediaPipe
|
||||
* Because different platforms may require different techniques for best performance, the API should allow flexibility in the way things are implemented behind the scenes.
|
||||
* A calculator should be allowed maximum flexibility in using the GPU for all or part of its operation, combining it with the CPU if necessary.
|
||||
|
||||
### OpenGL support
|
||||
MediaPipe supports OpenGL ES up to version 3.2 on Android and up to ES 3.0 on iOS. In addition, MediaPipe also supports Metal on iOS.
|
||||
## OpenGL ES Support
|
||||
|
||||
* MediaPipe allows graphs to run OpenGL in multiple GL contexts. For example, this can be very useful in graphs that combine a slower GPU inference path (eg, at 10 FPS) with a faster GPU rendering path (eg, at 30 FPS): since one GL context corresponds to one sequential command queue, using the same context for both tasks would reduce the rendering frame rate. One challenge MediaPipe's use of multiple contexts solves is the ability to communicate across them. An example scenario is one with an input video that is sent to both the rendering and inferences paths, and rendering needs to have access to the latest output from inference.
|
||||
MediaPipe supports OpenGL ES up to version 3.2 on Android/Linux and up to ES 3.0
|
||||
on iOS. In addition, MediaPipe also supports Metal on iOS.
|
||||
|
||||
* An OpenGL context cannot be accessed by multiple threads at the same time. Furthermore, switching the active GL context on the same thread can be slow on some Android devices. Therefore, our approach is to have one dedicated thread per context. Each thread issues GL commands, building up a serial command queue on its context, which is then executed by the GPU asynchronously.
|
||||
OpenGL ES 3.1 or greater is required (on Android/Linux systems) for running
|
||||
machine learning inference calculators and graphs.
|
||||
|
||||
### Life of a GPU calculator
|
||||
MediaPipe allows graphs to run OpenGL in multiple GL contexts. For example, this
|
||||
can be very useful in graphs that combine a slower GPU inference path (eg, at 10
|
||||
FPS) with a faster GPU rendering path (eg, at 30 FPS): since one GL context
|
||||
corresponds to one sequential command queue, using the same context for both
|
||||
tasks would reduce the rendering frame rate.
|
||||
|
||||
One challenge MediaPipe's use of multiple contexts solves is the ability to
|
||||
communicate across them. An example scenario is one with an input video that is
|
||||
sent to both the rendering and inferences paths, and rendering needs to have
|
||||
access to the latest output from inference.
|
||||
|
||||
An OpenGL context cannot be accessed by multiple threads at the same time.
|
||||
Furthermore, switching the active GL context on the same thread can be slow on
|
||||
some Android devices. Therefore, our approach is to have one dedicated thread
|
||||
per context. Each thread issues GL commands, building up a serial command queue
|
||||
on its context, which is then executed by the GPU asynchronously.
|
||||
|
||||
## Life of a GPU Calculator
|
||||
|
||||
This section presents the basic structure of the Process method of a GPU
|
||||
calculator derived from base class GlSimpleCalculator. The GPU calculator
|
||||
@@ -116,7 +141,7 @@ choices for MediaPipe GPU support:
|
||||
* Data that needs to be shared between all GPU-based calculators is provided as a external input that is implemented as a graph service and is managed by the `GlCalculatorHelper` class.
|
||||
* The combination of calculator-specific helpers and a shared graph service allows us great flexibility in managing the GPU resource: we can have a separate context per calculator, share a single context, share a lock or other synchronization primitives, etc. -- and all of this is managed by the helper and hidden from the individual calculators.
|
||||
|
||||
### GpuBuffer to ImageFrame converters
|
||||
## GpuBuffer to ImageFrame Converters
|
||||
|
||||
We provide two calculators called `GpuBufferToImageFrameCalculator` and `ImageFrameToGpuBufferCalculator`. These calculators convert between `ImageFrame` and `GpuBuffer`, allowing the construction of graphs that combine GPU and CPU calculators. They are supported on both iOS and Android
|
||||
|
||||
@@ -124,7 +149,15 @@ When possible, these calculators use platform-specific functionality to share da
|
||||
|
||||
The below diagram shows the data flow in a mobile application that captures video from the camera, runs it through a MediaPipe graph, and renders the output on the screen in real time. The dashed line indicates which parts are inside the MediaPipe graph proper. This application runs a Canny edge-detection filter on the CPU using OpenCV, and overlays it on top of the original video using the GPU.
|
||||
|
||||
|  |
|
||||
|:--:|
|
||||
| *Video frames from the camera are fed into the graph as `GpuBuffer` packets. The input stream is accessed by two calculators in parallel. `GpuBufferToImageFrameCalculator` converts the buffer into an `ImageFrame`, which is then sent through a grayscale converter and a canny filter (both based on OpenCV and running on the CPU), whose output is then converted into a `GpuBuffer` again. A multi-input GPU calculator, GlOverlayCalculator, takes as input both the original `GpuBuffer` and the one coming out of the edge detector, and overlays them using a shader. The output is then sent back to the application using a callback calculator, and the application renders the image to the screen using OpenGL.* |
|
||||

|
||||
|
||||
Video frames from the camera are fed into the graph as `GpuBuffer` packets. The
|
||||
input stream is accessed by two calculators in parallel.
|
||||
`GpuBufferToImageFrameCalculator` converts the buffer into an `ImageFrame`,
|
||||
which is then sent through a grayscale converter and a canny filter (both based
|
||||
on OpenCV and running on the CPU), whose output is then converted into a
|
||||
`GpuBuffer` again. A multi-input GPU calculator, GlOverlayCalculator, takes as
|
||||
input both the original `GpuBuffer` and the one coming out of the edge detector,
|
||||
and overlays them using a shader. The output is then sent back to the
|
||||
application using a callback calculator, and the application renders the image
|
||||
to the screen using OpenGL.
|
||||
@@ -0,0 +1,271 @@
|
||||
---
|
||||
layout: default
|
||||
title: Graphs
|
||||
parent: Framework Concepts
|
||||
nav_order: 2
|
||||
---
|
||||
|
||||
# Graphs
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## GraphConfig
|
||||
|
||||
A `GraphConfig` is a specification that describes the topology and functionality
|
||||
of a MediaPipe graph. In the specification, a node in the graph represents an
|
||||
instance of a particular calculator. All the necessary configurations of the
|
||||
node, such its type, inputs and outputs must be described in the specification.
|
||||
Description of the node can also include several optional fields, such as
|
||||
node-specific options, input policy and executor, discussed in
|
||||
[Synchronization](synchronization.md).
|
||||
|
||||
`GraphConfig` has several other fields to configure the global graph-level
|
||||
settings, eg, graph executor configs, number of threads, and maximum queue size
|
||||
of input streams. Several graph-level settings are useful for tuning the
|
||||
performance of the graph on different platforms (eg, desktop v.s. mobile). For
|
||||
instance, on mobile, attaching a heavy model-inference calculator to a separate
|
||||
executor can improve the performance of a real-time application since this
|
||||
enables thread locality.
|
||||
|
||||
Below is a trivial `GraphConfig` example where we have series of passthrough
|
||||
calculators :
|
||||
|
||||
```proto
|
||||
# This graph named main_pass_throughcals_nosubgraph.pbtxt contains 4
|
||||
# passthrough calculators.
|
||||
input_stream: "in"
|
||||
node {
|
||||
calculator: "PassThroughCalculator"
|
||||
input_stream: "in"
|
||||
output_stream: "out1"
|
||||
}
|
||||
node {
|
||||
calculator: "PassThroughCalculator"
|
||||
input_stream: "out1"
|
||||
output_stream: "out2"
|
||||
}
|
||||
node {
|
||||
calculator: "PassThroughCalculator"
|
||||
input_stream: "out2"
|
||||
output_stream: "out3"
|
||||
}
|
||||
node {
|
||||
calculator: "PassThroughCalculator"
|
||||
input_stream: "out3"
|
||||
output_stream: "out4"
|
||||
}
|
||||
```
|
||||
|
||||
## Subgraph
|
||||
|
||||
To modularize a `CalculatorGraphConfig` into sub-modules and assist with re-use
|
||||
of perception solutions, a MediaPipe graph can be defined as a `Subgraph`. The
|
||||
public interface of a subgraph consists of a set of input and output streams
|
||||
similar to a calculator's public interface. The subgraph can then be included in
|
||||
an `CalculatorGraphConfig` as if it were a calculator. When a MediaPipe graph is
|
||||
loaded from a `CalculatorGraphConfig`, each subgraph node is replaced by the
|
||||
corresponding graph of calculators. As a result, the semantics and performance
|
||||
of the subgraph is identical to the corresponding graph of calculators.
|
||||
|
||||
Below is an example of how to create a subgraph named `TwoPassThroughSubgraph`.
|
||||
|
||||
1. Defining the subgraph.
|
||||
|
||||
```proto
|
||||
# This subgraph is defined in two_pass_through_subgraph.pbtxt
|
||||
# and is registered as "TwoPassThroughSubgraph"
|
||||
|
||||
type: "TwoPassThroughSubgraph"
|
||||
input_stream: "out1"
|
||||
output_stream: "out3"
|
||||
|
||||
node {
|
||||
calculator: "PassThroughculator"
|
||||
input_stream: "out1"
|
||||
output_stream: "out2"
|
||||
}
|
||||
node {
|
||||
calculator: "PassThroughculator"
|
||||
input_stream: "out2"
|
||||
output_stream: "out3"
|
||||
}
|
||||
```
|
||||
|
||||
The public interface to the subgraph consists of:
|
||||
|
||||
* Graph input streams
|
||||
* Graph output streams
|
||||
* Graph input side packets
|
||||
* Graph output side packets
|
||||
|
||||
2. Register the subgraph using BUILD rule `mediapipe_simple_subgraph`. The
|
||||
parameter `register_as` defines the component name for the new subgraph.
|
||||
|
||||
```proto
|
||||
# Small section of BUILD file for registering the "TwoPassThroughSubgraph"
|
||||
# subgraph for use by main graph main_pass_throughcals.pbtxt
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "twopassthrough_subgraph",
|
||||
graph = "twopassthrough_subgraph.pbtxt",
|
||||
register_as = "TwoPassThroughSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:pass_through_calculator",
|
||||
"//mediapipe/framework:calculator_graph",
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
3. Use the subgraph in the main graph.
|
||||
|
||||
```proto
|
||||
# This main graph is defined in main_pass_throughcals.pbtxt
|
||||
# using subgraph called "TwoPassThroughSubgraph"
|
||||
|
||||
input_stream: "in"
|
||||
node {
|
||||
calculator: "PassThroughCalculator"
|
||||
input_stream: "in"
|
||||
output_stream: "out1"
|
||||
}
|
||||
node {
|
||||
calculator: "TwoPassThroughSubgraph"
|
||||
input_stream: "out1"
|
||||
output_stream: "out3"
|
||||
}
|
||||
node {
|
||||
calculator: "PassThroughCalculator"
|
||||
input_stream: "out3"
|
||||
output_stream: "out4"
|
||||
}
|
||||
```
|
||||
|
||||
## Cycles
|
||||
|
||||
<!-- TODO: add discussion of PreviousLoopbackCalculator -->
|
||||
|
||||
By default, MediaPipe requires calculator graphs to be acyclic and treats cycles
|
||||
in a graph as errors. If a graph is intended to have cycles, the cycles need to
|
||||
be annotated in the graph config. This page describes how to do that.
|
||||
|
||||
NOTE: The current approach is experimental and subject to change. We welcome
|
||||
your feedback.
|
||||
|
||||
Please use the `CalculatorGraphTest.Cycle` unit test in
|
||||
`mediapipe/framework/calculator_graph_test.cc` as sample code. Shown
|
||||
below is the cyclic graph in the test. The `sum` output of the adder is the sum
|
||||
of the integers generated by the integer source calculator.
|
||||
|
||||

|
||||
|
||||
This simple graph illustrates all the issues in supporting cyclic graphs.
|
||||
|
||||
### Back Edge Annotation
|
||||
|
||||
We require that an edge in each cycle be annotated as a back edge. This allows
|
||||
MediaPipe’s topological sort to work, after removing all the back edges.
|
||||
|
||||
There are usually multiple ways to select the back edges. Which edges are marked
|
||||
as back edges affects which nodes are considered as upstream and which nodes are
|
||||
considered as downstream, which in turn affects the priorities MediaPipe assigns
|
||||
to the nodes.
|
||||
|
||||
For example, the `CalculatorGraphTest.Cycle` test marks the `old_sum` edge as a
|
||||
back edge, so the Delay node is considered as a downstream node of the adder
|
||||
node and is given a higher priority. Alternatively, we could mark the `sum`
|
||||
input to the delay node as the back edge, in which case the delay node would be
|
||||
considered as an upstream node of the adder node and is given a lower priority.
|
||||
|
||||
### Initial Packet
|
||||
|
||||
For the adder calculator to be runnable when the first integer from the integer
|
||||
source arrives, we need an initial packet, with value 0 and with the same
|
||||
timestamp, on the `old_sum` input stream to the adder. This initial packet
|
||||
should be output by the delay calculator in the `Open()` method.
|
||||
|
||||
### Delay in a Loop
|
||||
|
||||
Each loop should incur a delay to align the previous `sum` output with the next
|
||||
integer input. This is also done by the delay node. So the delay node needs to
|
||||
know the following about the timestamps of the integer source calculator:
|
||||
|
||||
* The timestamp of the first output.
|
||||
|
||||
* The timestamp delta between successive outputs.
|
||||
|
||||
We plan to add an alternative scheduling policy that only cares about packet
|
||||
ordering and ignores packet timestamps, which will eliminate this inconvenience.
|
||||
|
||||
### Early Termination of a Calculator When One Input Stream is Done
|
||||
|
||||
By default, MediaPipe calls the `Close()` method of a non-source calculator when
|
||||
all of its input streams are done. In the example graph, we want to stop the
|
||||
adder node as soon as the integer source is done. This is accomplished by
|
||||
configuring the adder node with an alternative input stream handler,
|
||||
`EarlyCloseInputStreamHandler`.
|
||||
|
||||
### Relevant Source Code
|
||||
|
||||
#### Delay Calculator
|
||||
|
||||
Note the code in `Open()` that outputs the initial packet and the code in
|
||||
`Process()` that adds a (unit) delay to input packets. As noted above, this
|
||||
delay node assumes that its output stream is used alongside an input stream with
|
||||
packet timestamps 0, 1, 2, 3, ...
|
||||
|
||||
```c++
|
||||
class UnitDelayCalculator : public Calculator {
|
||||
public:
|
||||
static ::util::Status FillExpectations(
|
||||
const CalculatorOptions& extendable_options, PacketTypeSet* inputs,
|
||||
PacketTypeSet* outputs, PacketTypeSet* input_side_packets) {
|
||||
inputs->Index(0)->Set<int>("An integer.");
|
||||
outputs->Index(0)->Set<int>("The input delayed by one time unit.");
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::util::Status Open() final {
|
||||
Output()->Add(new int(0), Timestamp(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::util::Status Process() final {
|
||||
const Packet& packet = Input()->Value();
|
||||
Output()->AddPacket(packet.At(packet.Timestamp().NextAllowedInStream()));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
#### Graph Config
|
||||
|
||||
Note the `back_edge` annotation and the alternative `input_stream_handler`.
|
||||
|
||||
```proto
|
||||
node {
|
||||
calculator: 'GlobalCountSourceCalculator'
|
||||
input_side_packet: 'global_counter'
|
||||
output_stream: 'integers'
|
||||
}
|
||||
node {
|
||||
calculator: 'IntAdderCalculator'
|
||||
input_stream: 'integers'
|
||||
input_stream: 'old_sum'
|
||||
input_stream_info: {
|
||||
tag_index: ':1' # 'old_sum'
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: 'sum'
|
||||
input_stream_handler {
|
||||
input_stream_handler: 'EarlyCloseInputStreamHandler'
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: 'UnitDelayCalculator'
|
||||
input_stream: 'sum'
|
||||
output_stream: 'old_sum'
|
||||
}
|
||||
```
|
||||
@@ -1,10 +1,21 @@
|
||||
### Packets
|
||||
---
|
||||
layout: default
|
||||
title: Packets
|
||||
parent: Framework Concepts
|
||||
nav_order: 3
|
||||
---
|
||||
|
||||
- [Creating a packet](#creating-a-packet)
|
||||
# Packets
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
Each calculator is a node of of a graph. We describe how to create a new calculator, how to initialize a calculator, how to perform its calculations, input and output streams, timestamps, and options
|
||||
|
||||
#### Creating a packet
|
||||
## Creating a packet
|
||||
|
||||
Packets are generally created with `MediaPipe::Adopt()` (from packet.h).
|
||||
|
||||
```c++
|
||||
@@ -1,4 +1,16 @@
|
||||
# Framework Architecture
|
||||
---
|
||||
layout: default
|
||||
title: Synchronization
|
||||
parent: Framework Concepts
|
||||
nav_order: 4
|
||||
---
|
||||
|
||||
# Synchronization
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Scheduling mechanics
|
||||
|
||||
@@ -143,14 +155,13 @@ system that relaxes configured limits when needed.
|
||||
|
||||
The second system consists of inserting special nodes which can drop packets
|
||||
according to real-time constraints (typically using custom input policies)
|
||||
defined by [`RealTimeFlowLimiterCalculator`]. For example, a common pattern
|
||||
places a flow-control node at the input of a subgraph, with a loopback
|
||||
connection from the final output to the flow-control node. The flow-control node
|
||||
is thus able to keep track of how many timestamps are being processed in the
|
||||
downstream graph, and drop packets if this count hits a (configurable) limit;
|
||||
and since packets are dropped upstream, we avoid the wasted work that would
|
||||
result from partially processing a timestamp and then dropping packets between
|
||||
intermediate stages.
|
||||
defined by [`FlowLimiterCalculator`]. For example, a common pattern places a
|
||||
flow-control node at the input of a subgraph, with a loopback connection from
|
||||
the final output to the flow-control node. The flow-control node is thus able to
|
||||
keep track of how many timestamps are being processed in the downstream graph,
|
||||
and drop packets if this count hits a (configurable) limit; and since packets
|
||||
are dropped upstream, we avoid the wasted work that would result from partially
|
||||
processing a timestamp and then dropping packets between intermediate stages.
|
||||
|
||||
This calculator-based approach gives the graph author control of where packets
|
||||
can be dropped, and allows flexibility in adapting and customizing the graph’s
|
||||
@@ -161,4 +172,4 @@ behavior depending on resource constraints.
|
||||
[`SyncSetInputStreamHandler`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/stream_handler/sync_set_input_stream_handler.h
|
||||
[`ImmediateInputStreamHandler`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/stream_handler/immediate_input_stream_handler.h
|
||||
[`CalculatorGraphConfig::max_queue_size`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator.proto
|
||||
[`RealTimeFlowLimiterCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/real_time_flow_limiter_calculator.cc
|
||||
[`FlowLimiterCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/flow_limiter_calculator.cc
|
||||
@@ -0,0 +1,146 @@
|
||||
---
|
||||
layout: default
|
||||
title: MediaPipe Android Archive
|
||||
parent: Getting Started
|
||||
nav_order: 7
|
||||
---
|
||||
|
||||
# MediaPipe Android Archive
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
***Experimental Only***
|
||||
|
||||
The MediaPipe Android Archive (AAR) library is a convenient way to use MediaPipe
|
||||
with Android Studio and Gradle. MediaPipe doesn't publish a general AAR that can
|
||||
be used by all projects. Instead, developers need to add a mediapipe_aar()
|
||||
target to generate a custom AAR file for their own projects. This is necessary
|
||||
in order to include specific resources such as MediaPipe calculators needed for
|
||||
each project.
|
||||
|
||||
## Steps to build a MediaPipe AAR
|
||||
|
||||
1. Create a mediapipe_aar() target.
|
||||
|
||||
In the MediaPipe directory, create a new mediapipe_aar() target in a BUILD
|
||||
file. You need to figure out what calculators are used in the graph and
|
||||
provide the calculator dependencies to the mediapipe_aar(). For example, to
|
||||
build an AAR for [MediaPipe Face Detection](../solutions/face_detection.md),
|
||||
you can put the following code into
|
||||
mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/BUILD.
|
||||
|
||||
```
|
||||
load("//mediapipe/java/com/google/mediapipe:mediapipe_aar.bzl", "mediapipe_aar")
|
||||
|
||||
mediapipe_aar(
|
||||
name = "mp_face_detection_aar",
|
||||
calculators = ["//mediapipe/graphs/face_detection:mobile_calculators"],
|
||||
)
|
||||
```
|
||||
|
||||
2. Run the Bazel build command to generate the AAR.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
|
||||
//path/to/the/aar/build/file:aar_name
|
||||
```
|
||||
|
||||
For the face detection AAR target we made in the step 1, run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
|
||||
//mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar
|
||||
|
||||
# It should print:
|
||||
# Target //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar up-to-date:
|
||||
# bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
|
||||
```
|
||||
|
||||
3. (Optional) Save the AAR to your preferred location.
|
||||
|
||||
```bash
|
||||
cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
|
||||
/absolute/path/to/your/preferred/location
|
||||
```
|
||||
|
||||
## Steps to use a MediaPipe AAR in Android Studio with Gradle
|
||||
|
||||
1. Start Android Studio and go to your project.
|
||||
|
||||
2. Copy the AAR into app/libs.
|
||||
|
||||
```bash
|
||||
cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
|
||||
/path/to/your/app/libs/
|
||||
```
|
||||
|
||||

|
||||
|
||||
3. Make app/src/main/assets and copy assets (graph, model, and etc) into
|
||||
app/src/main/assets.
|
||||
|
||||
Build the MediaPipe binary graph and copy the assets into
|
||||
app/src/main/assets, e.g., for the face detection graph, you need to build
|
||||
and copy
|
||||
[the binary graph](https://github.com/google/mediapipe/blob/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD#L41),
|
||||
[the tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
|
||||
and
|
||||
[the label map](https://github.com/google/mediapipe/blob/master/mediapipe/models/face_detection_front_labelmap.txt).
|
||||
|
||||
```bash
|
||||
bazel build -c opt mediapipe/mediapipe/graphs/face_detection:mobile_gpu_binary_graph
|
||||
cp bazel-bin/mediapipe/graphs/face_detection/mobile_gpu.binarypb /path/to/your/app/src/main/assets/
|
||||
cp mediapipe/models/face_detection_front.tflite /path/to/your/app/src/main/assets/
|
||||
cp mediapipe/models/face_detection_front_labelmap.txt /path/to/your/app/src/main/assets/
|
||||
```
|
||||
|
||||

|
||||
|
||||
4. Make app/src/main/jniLibs and copy OpenCV JNI libraries into
|
||||
app/src/main/jniLibs.
|
||||
|
||||
MediaPipe depends on OpenCV, you will need to copy the precompiled OpenCV so
|
||||
files into app/src/main/jniLibs. You can download the official OpenCV
|
||||
Android SDK from
|
||||
[here](https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip)
|
||||
and run:
|
||||
|
||||
```bash
|
||||
cp -R ~/Downloads/OpenCV-android-sdk/sdk/native/libs/arm* /path/to/your/app/src/main/jniLibs/
|
||||
```
|
||||
|
||||

|
||||
|
||||
5. Modify app/build.gradle to add MediaPipe dependencies and MediaPipe AAR.
|
||||
|
||||
```
|
||||
dependencies {
|
||||
implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar'])
|
||||
implementation 'androidx.appcompat:appcompat:1.0.2'
|
||||
implementation 'androidx.constraintlayout:constraintlayout:1.1.3'
|
||||
testImplementation 'junit:junit:4.12'
|
||||
androidTestImplementation 'androidx.test.ext:junit:1.1.0'
|
||||
androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1'
|
||||
// MediaPipe deps
|
||||
implementation 'com.google.flogger:flogger:0.3.1'
|
||||
implementation 'com.google.flogger:flogger-system-backend:0.3.1'
|
||||
implementation 'com.google.code.findbugs:jsr305:3.0.2'
|
||||
implementation 'com.google.guava:guava:27.0.1-android'
|
||||
implementation 'com.google.guava:guava:27.0.1-android'
|
||||
implementation 'com.google.protobuf:protobuf-java:3.11.4'
|
||||
// CameraX core library
|
||||
def camerax_version = "1.0.0-alpha06"
|
||||
implementation "androidx.camera:camera-core:$camerax_version"
|
||||
implementation "androidx.camera:camera-camera2:$camerax_version"
|
||||
}
|
||||
```
|
||||
|
||||
6. Follow our Android app examples to use MediaPipe in Android Studio for your
|
||||
use case. If you are looking for an example, a face detection example can be
|
||||
found
|
||||
[here](https://github.com/jiuqiant/mediapipe_face_detection_aar_example) and
|
||||
a multi-hand tracking example can be found
|
||||
[here](https://github.com/jiuqiant/mediapipe_multi_hands_tracking_aar_example).
|
||||
@@ -0,0 +1,347 @@
|
||||
---
|
||||
layout: default
|
||||
title: Building MediaPipe Examples
|
||||
parent: Getting Started
|
||||
nav_order: 2
|
||||
---
|
||||
|
||||
# Building MediaPipe Examples
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Android
|
||||
|
||||
### Prerequisite
|
||||
|
||||
* Java Runtime.
|
||||
* Android SDK release 28.0.3 and above.
|
||||
* Android NDK r18b and above.
|
||||
|
||||
MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
|
||||
below for Android Studio setup). However, if you prefer using MediaPipe without
|
||||
Android Studio, please run
|
||||
[`setup_android_sdk_and_ndk.sh`](https://github.com/google/mediapipe/blob/master/setup_android_sdk_and_ndk.sh)
|
||||
to download and setup Android SDK and NDK before building any Android example
|
||||
apps.
|
||||
|
||||
If Android SDK and NDK are already installed (e.g., by Android Studio), set
|
||||
$ANDROID_HOME and $ANDROID_NDK_HOME to point to the installed SDK and NDK.
|
||||
|
||||
```bash
|
||||
export ANDROID_HOME=<path to the Android SDK>
|
||||
export ANDROID_NDK_HOME=<path to the Android NDK>
|
||||
```
|
||||
|
||||
In order to use MediaPipe on earlier Android versions, MediaPipe needs to switch
|
||||
to a lower Android API level. You can achieve this by specifying `api_level =
|
||||
$YOUR_INTENDED_API_LEVEL` in android_ndk_repository() and/or
|
||||
android_sdk_repository() in the
|
||||
[`WORKSPACE`](https://github.com/google/mediapipe/blob/master/WORKSPACE) file.
|
||||
|
||||
Please verify all the necessary packages are installed.
|
||||
|
||||
* Android SDK Platform API Level 28 or 29
|
||||
* Android SDK Build-Tools 28 or 29
|
||||
* Android SDK Platform-Tools 28 or 29
|
||||
* Android SDK Tools 26.1.1
|
||||
* Android NDK 17c or above
|
||||
|
||||
### Option 1: Build with Bazel in Command Line
|
||||
|
||||
Tip: You can run this
|
||||
[script](https://github.com/google/mediapipe/blob/master/build_android_examples.sh)
|
||||
to build (and install) all MediaPipe Android example apps.
|
||||
|
||||
1. To build an Android example app, build against the corresponding
|
||||
`android_binary` build target. For instance, for
|
||||
[MediaPipe Hands](../solutions/hands.md) the target is `handtrackinggpu` in
|
||||
the
|
||||
[BUILD](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD)
|
||||
file:
|
||||
|
||||
Note: To reduce the binary size, consider appending `--linkopt="-s"` to the
|
||||
command below to strip symbols.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu:handtrackinggpu
|
||||
```
|
||||
|
||||
2. Install it on a device with:
|
||||
|
||||
```bash
|
||||
adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/handtrackinggpu.apk
|
||||
```
|
||||
|
||||
### Option 2: Build with Bazel in Android Studio
|
||||
|
||||
The MediaPipe project can be imported into Android Studio using the Bazel
|
||||
plugins. This allows the MediaPipe examples to be built and modified in Android
|
||||
Studio.
|
||||
|
||||
To incorporate MediaPipe into an existing Android Studio project, see these
|
||||
[instructions](./android_archive_library.md) that use Android Archive (AAR) and
|
||||
Gradle.
|
||||
|
||||
The steps below use Android Studio 3.5 to build and install a MediaPipe example
|
||||
app:
|
||||
|
||||
1. Install and launch Android Studio 3.5.
|
||||
|
||||
2. Select `Configure` -> `SDK Manager` -> `SDK Platforms`.
|
||||
|
||||
* Verify that Android SDK Platform API Level 28 or 29 is installed.
|
||||
* Take note of the Android SDK Location, e.g.,
|
||||
`/usr/local/home/Android/Sdk`.
|
||||
|
||||
3. Select `Configure` -> `SDK Manager` -> `SDK Tools`.
|
||||
|
||||
* Verify that Android SDK Build-Tools 28 or 29 is installed.
|
||||
* Verify that Android SDK Platform-Tools 28 or 29 is installed.
|
||||
* Verify that Android SDK Tools 26.1.1 is installed.
|
||||
* Verify that Android NDK 17c or above is installed.
|
||||
* Take note of the Android NDK Location, e.g.,
|
||||
`/usr/local/home/Android/Sdk/ndk-bundle` or
|
||||
`/usr/local/home/Android/Sdk/ndk/20.0.5594570`.
|
||||
|
||||
4. Set environment variables `$ANDROID_HOME` and `$ANDROID_NDK_HOME` to point
|
||||
to the installed SDK and NDK.
|
||||
|
||||
```bash
|
||||
export ANDROID_HOME=/usr/local/home/Android/Sdk
|
||||
|
||||
# If the NDK libraries are installed by a previous version of Android Studio, do
|
||||
export ANDROID_NDK_HOME=/usr/local/home/Android/Sdk/ndk-bundle
|
||||
# If the NDK libraries are installed by Android Studio 3.5, do
|
||||
export ANDROID_NDK_HOME=/usr/local/home/Android/Sdk/ndk/<version number>
|
||||
```
|
||||
|
||||
5. Select `Configure` -> `Plugins` to install `Bazel`.
|
||||
|
||||
6. On Linux, select `File` -> `Settings` -> `Bazel settings`. On macos, select
|
||||
`Android Studio` -> `Preferences` -> `Bazel settings`. Then, modify `Bazel
|
||||
binary location` to be the same as the output of `$ which bazel`.
|
||||
|
||||
7. Select `Import Bazel Project`.
|
||||
|
||||
* Select `Workspace`: `/path/to/mediapipe` and select `Next`.
|
||||
* Select `Generate from BUILD file`: `/path/to/mediapipe/BUILD` and select
|
||||
`Next`.
|
||||
* Modify `Project View` to be the following and select `Finish`.
|
||||
|
||||
```
|
||||
directories:
|
||||
# read project settings, e.g., .bazelrc
|
||||
.
|
||||
-mediapipe/objc
|
||||
-mediapipe/examples/ios
|
||||
|
||||
targets:
|
||||
//mediapipe/examples/android/...:all
|
||||
//mediapipe/java/...:all
|
||||
|
||||
android_sdk_platform: android-29
|
||||
|
||||
sync_flags:
|
||||
--host_crosstool_top=@bazel_tools//tools/cpp:toolchain
|
||||
```
|
||||
|
||||
8. Select `Bazel` -> `Sync` -> `Sync project with Build files`.
|
||||
|
||||
Note: Even after doing step 4, if you still see the error: `"no such package
|
||||
'@androidsdk//': Either the path attribute of android_sdk_repository or the
|
||||
ANDROID_HOME environment variable must be set."`, please modify the
|
||||
[`WORKSPACE`](https://github.com/google/mediapipe/blob/master/WORKSPACE)
|
||||
file to point to your SDK and NDK library locations, as below:
|
||||
|
||||
```
|
||||
android_sdk_repository(
|
||||
name = "androidsdk",
|
||||
path = "/path/to/android/sdk"
|
||||
)
|
||||
|
||||
android_ndk_repository(
|
||||
name = "androidndk",
|
||||
path = "/path/to/android/ndk"
|
||||
)
|
||||
```
|
||||
|
||||
9. Connect an Android device to the workstation.
|
||||
|
||||
10. Select `Run...` -> `Edit Configurations...`.
|
||||
|
||||
* Select `Templates` -> `Bazel Command`.
|
||||
* Enter Target Expression:
|
||||
`//mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu:handtrackinggpu`
|
||||
* Enter Bazel command: `mobile-install`.
|
||||
* Enter Bazel flags: `-c opt --config=android_arm64`.
|
||||
* Press the `[+]` button to add the new configuration.
|
||||
* Select `Run` to run the example app on the connected Android device.
|
||||
|
||||
## iOS
|
||||
|
||||
### Prerequisite
|
||||
|
||||
1. Install [Xcode](https://developer.apple.com/xcode/), and additionally
|
||||
install the Command Line Tools by:
|
||||
|
||||
```bash
|
||||
xcode-select --install
|
||||
```
|
||||
|
||||
2. Install [Bazel](https://bazel.build/).
|
||||
|
||||
We recommend using [Homebrew](https://brew.sh/) to get the latest version.
|
||||
|
||||
3. Set Python 3.7 as the default Python version and install the Python "six"
|
||||
library.
|
||||
|
||||
To make Mediapipe work with TensorFlow, please set Python 3.7 as the default
|
||||
Python version and install the Python "six" library.
|
||||
|
||||
```bash
|
||||
pip3 install --user six
|
||||
```
|
||||
|
||||
4. Follow
|
||||
[Apple's instructions](https://developer.apple.com/support/certificates/) to
|
||||
obtain the required development certificates and provisioning profiles for
|
||||
your iOS device.
|
||||
|
||||
Tip: You can the following command to see the provisioning profiles you have
|
||||
previously downloaded using Xcode: `open
|
||||
~/Library/MobileDevice/"Provisioning Profiles"`. If there are none, generate
|
||||
and download a profile on
|
||||
[Apple's developer site](https://developer.apple.com/account/resources/).
|
||||
|
||||
5. Clone the MediaPipe repository.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/google/mediapipe.git
|
||||
```
|
||||
|
||||
6. In the cloned MediaPipe repository, symlink or copy your provisioning profile
|
||||
to `mediapipe/provisioning_profile.mobileprovision`, e.g.,
|
||||
|
||||
```bash
|
||||
cd mediapipe
|
||||
ln -s ~/Downloads/MyProvisioningProfile.mobileprovision mediapipe/provisioning_profile.mobileprovision
|
||||
```
|
||||
|
||||
### Option 1: Build with Bazel in Command Line
|
||||
|
||||
1. Modify the `bundle_id` field of the app's `ios_application` build target to
|
||||
use your own identifier. For instance, for
|
||||
[MediaPipe Hands](../solutions/hands.md), the `bundle_id` is in the
|
||||
`HandTrackingGpuApp` target in the
|
||||
[BUILD](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handtrackinggpu/BUILD)
|
||||
file.
|
||||
|
||||
2. Again using [MediaPipe Hands](../solutions/hands.md) for example, run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp
|
||||
```
|
||||
|
||||
You may see a permission request from `codesign` in order to sign the app.
|
||||
|
||||
Tip: You can run this
|
||||
[script](https://github.com/google/mediapipe/blob/master/build_ios_examples.sh)
|
||||
to build all MediaPipe iOS example apps.
|
||||
|
||||
3. In Xcode, open the `Devices and Simulators` window (command-shift-2).
|
||||
|
||||
4. Make sure your device is connected. You will see a list of installed apps.
|
||||
Press the "+" button under the list, and select the `.ipa` file built by
|
||||
Bazel.
|
||||
|
||||
5. You can now run the app on your device.
|
||||
|
||||
### Option 2: Build in Xcode
|
||||
|
||||
Note: This workflow requires a separate tool in addition to Bazel. If it fails
|
||||
to work for some reason, please resort to the command-line build instructions in
|
||||
the previous section.
|
||||
|
||||
1. We will use a tool called [Tulsi](https://tulsi.bazel.build/) for generating
|
||||
Xcode projects from Bazel build configurations.
|
||||
|
||||
```bash
|
||||
# cd out of the mediapipe directory, then:
|
||||
git clone https://github.com/bazelbuild/tulsi.git
|
||||
cd tulsi
|
||||
# remove Xcode version from Tulsi's .bazelrc (see http://github.com/bazelbuild/tulsi#building-and-installing):
|
||||
sed -i .orig '/xcode_version/d' .bazelrc
|
||||
# build and run Tulsi:
|
||||
sh build_and_run.sh
|
||||
```
|
||||
|
||||
This will install `Tulsi.app` inside the `Applications` directory in your
|
||||
home directory.
|
||||
|
||||
2. Open `mediapipe/Mediapipe.tulsiproj` using the Tulsi app.
|
||||
|
||||
Important: If Tulsi displays an error saying "Bazel could not be found",
|
||||
press the "Bazel..." button in the Packages tab and select the `bazel`
|
||||
executable in your homebrew `/bin/` directory.
|
||||
|
||||
3. Select the MediaPipe config in the Configs tab, then press the Generate
|
||||
button below. You will be asked for a location to save the Xcode project.
|
||||
Once the project is generated, it will be opened in Xcode.
|
||||
|
||||
4. You can now select any of the MediaPipe demos in the target menu, and build
|
||||
and run them as normal.
|
||||
|
||||
Note: When you ask Xcode to run an app, by default it will use the Debug
|
||||
configuration. Some of our demos are computationally heavy; you may want to
|
||||
use the Release configuration for better performance.
|
||||
|
||||
Tip: To switch build configuration in Xcode, click on the target menu,
|
||||
choose "Edit Scheme...", select the Run action, and switch the Build
|
||||
Configuration from Debug to Release. Note that this is set independently for
|
||||
each target.
|
||||
|
||||
## Desktop
|
||||
|
||||
### Option 1: Running on CPU
|
||||
|
||||
1. To build, for example, [MediaPipe Hands](../solutions/hands.md), run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu
|
||||
```
|
||||
|
||||
This will open up your webcam as long as it is connected and on. Any errors
|
||||
is likely due to your webcam being not accessible.
|
||||
|
||||
2. To run the application:
|
||||
|
||||
```bash
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_cpu \
|
||||
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt
|
||||
```
|
||||
|
||||
### Option 2: Running on GPU
|
||||
|
||||
Note: This currently works only on Linux, and please first follow
|
||||
[OpenGL ES Setup on Linux Desktop](./gpu_support.md#opengl-es-setup-on-linux-desktop).
|
||||
|
||||
1. To build, for example, [MediaPipe Hands](../solutions/hands.md), run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
|
||||
mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu
|
||||
```
|
||||
|
||||
This will open up your webcam as long as it is connected and on. Any errors
|
||||
is likely due to your webcam being not accessible, or GPU drivers not setup
|
||||
properly.
|
||||
|
||||
2. To run the application:
|
||||
|
||||
```bash
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_gpu \
|
||||
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt
|
||||
```
|
||||
@@ -1,12 +1,16 @@
|
||||
## Questions and Answers
|
||||
---
|
||||
layout: default
|
||||
title: FAQ
|
||||
parent: Getting Started
|
||||
nav_order: 9
|
||||
---
|
||||
|
||||
- [How to convert ImageFrames and GpuBuffers](#how-to-convert-imageframes-and-gpubuffers)
|
||||
- [How to visualize perceived results](#how-to-visualize-perception-results)
|
||||
- [How to run calculators in parallel](#how-to-run-calculators-in-parallel)
|
||||
- [Output timestamps when using ImmediateInputStreamHandler](#output-timestamps-when-using-immediateinputstreamhandler)
|
||||
- [How to change settings at runtime](#how-to-change-settings-at-runtime)
|
||||
- [How to process real-time input streams](#how-to-process-real-time-input-streams)
|
||||
- [Can I run MediaPipe on MS Windows?](#can-i-run-mediapipe-on-ms-windows)
|
||||
# FAQ
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
### How to convert ImageFrames and GpuBuffers
|
||||
|
||||
@@ -16,10 +20,10 @@ type [`ImageFrame`] and [`GpuBuffer`]. [`ImageFrame`] refers to image data in
|
||||
CPU memory in any of a number of bitmap image formats. [`GpuBuffer`] refers to
|
||||
image data in GPU memory. You can find more detail in the Framework Concepts
|
||||
section
|
||||
[GpuBuffer to ImageFrame converters](./gpu.md).
|
||||
[GpuBuffer to ImageFrame Converters](./gpu.md#gpubuffer-to-imageframe-converters).
|
||||
You can see an example in:
|
||||
|
||||
* [`object_detection_android_cpu.pbtxt`]
|
||||
* [`object_detection_mobile_cpu.pbtxt`]
|
||||
|
||||
### How to visualize perception results
|
||||
|
||||
@@ -29,7 +33,7 @@ the recognized objects. The results can be displayed in a diagnostic window when
|
||||
running on a workstation, or in a texture frame when running on device. You can
|
||||
see an example use of [`AnnotationOverlayCalculator`] in:
|
||||
|
||||
* [`face_detection_android_gpu.pbtxt`].
|
||||
* [`face_detection_mobile_gpu.pbtxt`].
|
||||
|
||||
### How to run calculators in parallel
|
||||
|
||||
@@ -106,7 +110,7 @@ continues as long as necessary. For online processing, it is often necessary to
|
||||
drop input packets in order to keep pace with the arrival of input data frames.
|
||||
When inputs arrive too frequently, the recommended technique for dropping
|
||||
packets is to use the MediaPipe calculators designed specifically for this
|
||||
purpose such as [`RealTimeFlowLimiterCalculator`] and [`PacketClonerCalculator`].
|
||||
purpose such as [`FlowLimiterCalculator`] and [`PacketClonerCalculator`].
|
||||
|
||||
For online processing, it is also necessary to promptly determine when processing
|
||||
can proceed. MediaPipe supports this by propagating timestamp bounds between
|
||||
@@ -124,20 +128,19 @@ MacOS, Android, and iOS. The core of MediaPipe framework is a C++ library
|
||||
conforming to the C++11 standard, so it is relatively easy to port to
|
||||
additional platforms.
|
||||
|
||||
[`object_detection_android_cpu.pbtxt`]: https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_android_cpu.pbtxt
|
||||
|
||||
[`object_detection_mobile_cpu.pbtxt`]: https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt
|
||||
[`ImageFrame`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/formats/image_frame.h
|
||||
[`GpuBuffer`]: https://github.com/google/mediapipe/tree/master/mediapipe/gpu/gpu_buffer.h
|
||||
[`GpuBufferToImageFrameCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/gpu/gpu_buffer_to_image_frame_calculator.cc
|
||||
[`ImageFrameToGpuBufferCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc
|
||||
[`AnnotationOverlayCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/util/annotation_overlay_calculator.cc
|
||||
[`face_detection_android_gpu.pbtxt`]: https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_detection/face_detection_android_gpu.pbtxt
|
||||
[`face_detection_mobile_gpu.pbtxt`]: https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt
|
||||
[`CalculatorBase::Process`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.h
|
||||
[`max_in_flight`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator.proto
|
||||
[`RoundRobinDemuxCalculator`]: https://github.com/google/mediapipe/tree/master//mediapipe/calculators/core/round_robin_demux_calculator.cc
|
||||
[`ScaleImageCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/image/scale_image_calculator.cc
|
||||
[`ImmediateInputStreamHandler`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc
|
||||
[`CalculatorGraphConfig`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator.proto
|
||||
[`RealTimeFlowLimiterCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/real_time_flow_limiter_calculator.cc
|
||||
[`FlowLimiterCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/flow_limiter_calculator.cc
|
||||
[`PacketClonerCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/packet_cloner_calculator.cc
|
||||
[`MakePairCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/make_pair_calculator.cc
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
layout: default
|
||||
title: Getting Started
|
||||
nav_order: 2
|
||||
has_children: true
|
||||
---
|
||||
|
||||
# Getting Started
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
@@ -0,0 +1,186 @@
|
||||
---
|
||||
layout: default
|
||||
title: GPU Support
|
||||
parent: Getting Started
|
||||
nav_order: 6
|
||||
---
|
||||
|
||||
# GPU Support
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## OpenGL ES Support
|
||||
|
||||
MediaPipe supports OpenGL ES up to version 3.2 on Android/Linux and up to ES 3.0
|
||||
on iOS. In addition, MediaPipe also supports Metal on iOS.
|
||||
|
||||
OpenGL ES 3.1 or greater is required (on Android/Linux systems) for running
|
||||
machine learning inference calculators and graphs.
|
||||
|
||||
## Disable OpenGL ES Support
|
||||
|
||||
By default, building MediaPipe (with no special bazel flags) attempts to compile
|
||||
and link against OpenGL ES (and for iOS also Metal) libraries.
|
||||
|
||||
On platforms where OpenGL ES is not available (see also
|
||||
[OpenGL ES Setup on Linux Desktop](#opengl-es-setup-on-linux-desktop)), you
|
||||
should disable OpenGL ES support with:
|
||||
|
||||
```
|
||||
$ bazel build --define MEDIAPIPE_DISABLE_GPU=1 <my-target>
|
||||
```
|
||||
|
||||
Note: On Android and iOS, OpenGL ES is required by MediaPipe framework and the
|
||||
support should never be disabled.
|
||||
|
||||
## OpenGL ES Setup on Linux Desktop
|
||||
|
||||
On Linux desktop with video cards that support OpenGL ES 3.1+, MediaPipe can run
|
||||
GPU compute and rendering and perform TFLite inference on GPU.
|
||||
|
||||
To check if your Linux desktop GPU can run MediaPipe with OpenGL ES:
|
||||
|
||||
```bash
|
||||
$ sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
|
||||
$ sudo apt-get install mesa-utils
|
||||
$ glxinfo | grep -i opengl
|
||||
```
|
||||
|
||||
For example, it may print:
|
||||
|
||||
```bash
|
||||
$ glxinfo | grep -i opengl
|
||||
...
|
||||
OpenGL ES profile version string: OpenGL ES 3.2 NVIDIA 430.50
|
||||
OpenGL ES profile shading language version string: OpenGL ES GLSL ES 3.20
|
||||
OpenGL ES profile extensions:
|
||||
```
|
||||
|
||||
*Notice the ES 3.20 text above.*
|
||||
|
||||
You need to see ES 3.1 or greater printed in order to perform TFLite inference
|
||||
on GPU in MediaPipe. With this setup, build with:
|
||||
|
||||
```
|
||||
$ bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 <my-target>
|
||||
```
|
||||
|
||||
If only ES 3.0 or below is supported, you can still build MediaPipe targets that
|
||||
don't require TFLite inference on GPU with:
|
||||
|
||||
```
|
||||
$ bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE <my-target>
|
||||
```
|
||||
|
||||
Note: MEDIAPIPE_DISABLE_GL_COMPUTE is already defined automatically on all Apple
|
||||
systems (Apple doesn't support OpenGL ES 3.1+).
|
||||
|
||||
## TensorFlow CUDA Support and Setup on Linux Desktop
|
||||
|
||||
MediaPipe framework doesn't require CUDA for GPU compute and rendering. However,
|
||||
MediaPipe can work with TensorFlow to perform GPU inference on video cards that
|
||||
support CUDA.
|
||||
|
||||
To enable TensorFlow GPU inference with MediaPipe, the first step is to follow
|
||||
the
|
||||
[TensorFlow GPU documentation](https://www.tensorflow.org/install/gpu#software_requirements)
|
||||
to install the required NVIDIA software on your Linux desktop.
|
||||
|
||||
After installation, update `$PATH` and `$LD_LIBRARY_PATH` and run `ldconfig`
|
||||
with:
|
||||
|
||||
```
|
||||
$ export PATH=/usr/local/cuda-10.1/bin${PATH:+:${PATH}}
|
||||
$ export LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64,/usr/local/cuda-10.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
|
||||
$ sudo ldconfig
|
||||
```
|
||||
|
||||
It's recommended to verify the installation of CUPTI, CUDA, CuDNN, and NVCC:
|
||||
|
||||
```
|
||||
$ ls /usr/local/cuda/extras/CUPTI
|
||||
/lib64
|
||||
libcupti.so libcupti.so.10.1.208 libnvperf_host.so libnvperf_target.so
|
||||
libcupti.so.10.1 libcupti_static.a libnvperf_host_static.a
|
||||
|
||||
$ ls /usr/local/cuda-10.1
|
||||
LICENSE bin extras lib64 libnvvp nvml samples src tools
|
||||
README doc include libnsight nsightee_plugins nvvm share targets version.txt
|
||||
|
||||
$ nvcc -V
|
||||
nvcc: NVIDIA (R) Cuda compiler driver
|
||||
Copyright (c) 2005-2019 NVIDIA Corporation
|
||||
Built on Sun_Jul_28_19:07:16_PDT_2019
|
||||
Cuda compilation tools, release 10.1, V10.1.243
|
||||
|
||||
$ ls /usr/lib/x86_64-linux-gnu/ | grep libcudnn.so
|
||||
libcudnn.so
|
||||
libcudnn.so.7
|
||||
libcudnn.so.7.6.4
|
||||
```
|
||||
|
||||
Setting `$TF_CUDA_PATHS` is the way to declare where the CUDA library is. Note
|
||||
that the following code snippet also adds `/usr/lib/x86_64-linux-gnu` and
|
||||
`/usr/include` into `$TF_CUDA_PATHS` for cudablas and libcudnn.
|
||||
|
||||
```
|
||||
$ export TF_CUDA_PATHS=/usr/local/cuda-10.1,/usr/lib/x86_64-linux-gnu,/usr/include
|
||||
```
|
||||
|
||||
To make MediaPipe get TensorFlow's CUDA settings, find TensorFlow's
|
||||
[.bazelrc](https://github.com/tensorflow/tensorflow/blob/master/.bazelrc) and
|
||||
copy the `build:using_cuda` and `build:cuda` section into MediaPipe's .bazelrc
|
||||
file. For example, as of April 23, 2020, TensorFlow's CUDA setting is the
|
||||
following:
|
||||
|
||||
```
|
||||
# This config refers to building with CUDA available. It does not necessarily
|
||||
# mean that we build CUDA op kernels.
|
||||
build:using_cuda --define=using_cuda=true
|
||||
build:using_cuda --action_env TF_NEED_CUDA=1
|
||||
build:using_cuda --crosstool_top=@local_config_cuda//crosstool:toolchain
|
||||
|
||||
# This config refers to building CUDA op kernels with nvcc.
|
||||
build:cuda --config=using_cuda
|
||||
build:cuda --define=using_cuda_nvcc=true
|
||||
```
|
||||
|
||||
Finally, build MediaPipe with TensorFlow GPU with two more flags `--config=cuda`
|
||||
and `--spawn_strategy=local`. For example:
|
||||
|
||||
```
|
||||
$ bazel build -c opt --config=cuda --spawn_strategy=local \
|
||||
--define no_aws_support=true --copt -DMESA_EGL_NO_X11_HEADERS \
|
||||
mediapipe/examples/desktop/object_detection:object_detection_tensorflow
|
||||
```
|
||||
|
||||
While the binary is running, it prints out the GPU device info:
|
||||
|
||||
```
|
||||
I external/org_tensorflow/tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1
|
||||
I external/org_tensorflow/tensorflow/core/common_runtime/gpu/gpu_device.cc:1544] Found device 0 with properties: pciBusID: 0000:00:04.0 name: Tesla T4 computeCapability: 7.5 coreClock: 1.59GHz coreCount: 40 deviceMemorySize: 14.75GiB deviceMemoryBandwidth: 298.08GiB/s
|
||||
I external/org_tensorflow/tensorflow/core/common_runtime/gpu/gpu_device.cc:1686] Adding visible gpu devices: 0
|
||||
```
|
||||
|
||||
You can monitor the GPU usage to verify whether the GPU is used for model
|
||||
inference.
|
||||
|
||||
```
|
||||
$ nvidia-smi --query-gpu=utilization.gpu --format=csv --loop=1
|
||||
|
||||
0 %
|
||||
0 %
|
||||
4 %
|
||||
5 %
|
||||
83 %
|
||||
21 %
|
||||
22 %
|
||||
27 %
|
||||
29 %
|
||||
100 %
|
||||
0 %
|
||||
0%
|
||||
```
|
||||
@@ -1,4 +1,16 @@
|
||||
# Hello World! in MediaPipe on Android
|
||||
---
|
||||
layout: default
|
||||
title: Hello World! on Android
|
||||
parent: Getting Started
|
||||
nav_order: 3
|
||||
---
|
||||
|
||||
# Hello World! on Android
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Introduction
|
||||
|
||||
@@ -14,31 +26,39 @@ graph on Android.
|
||||
A simple camera app for real-time Sobel edge detection applied to a live video
|
||||
stream on an Android device.
|
||||
|
||||
{width="300"}
|
||||

|
||||
|
||||
## Setup
|
||||
|
||||
1. Install MediaPipe on your system, see [MediaPipe installation guide] for
|
||||
details.
|
||||
2. Install Android Development SDK and Android NDK. See how to do so in
|
||||
[Setting up Android SDK and NDK].
|
||||
2. Install Android Development SDK and Android NDK. See how to do so also in
|
||||
[MediaPipe installation guide].
|
||||
3. Enable [developer options] on your Android device.
|
||||
4. Setup [Bazel] on your system to build and deploy the Android app.
|
||||
|
||||
## Graph for edge detection
|
||||
|
||||
We will be using the following graph, [`edge_detection_android_gpu.pbtxt`]:
|
||||
We will be using the following graph, [`edge_detection_mobile_gpu.pbtxt`]:
|
||||
|
||||
```
|
||||
# MediaPipe graph that performs GPU Sobel edge detection on a live video stream.
|
||||
# Used in the examples
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/basic.
|
||||
# mediapipe/examples/ios/edgedetectiongpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
output_stream: "output_video"
|
||||
|
||||
# Converts RGB images into luminance images, still stored in RGB format.
|
||||
node: {
|
||||
calculator: "LuminanceCalculator"
|
||||
input_stream: "input_video"
|
||||
output_stream: "luma_video"
|
||||
}
|
||||
|
||||
# Applies the Sobel filter to luminance images sotred in RGB format.
|
||||
node: {
|
||||
calculator: "SobelEdgesCalculator"
|
||||
input_stream: "luma_video"
|
||||
@@ -48,7 +68,7 @@ node: {
|
||||
|
||||
A visualization of the graph is shown below:
|
||||
|
||||
{width="200"}
|
||||

|
||||
|
||||
This graph has a single input stream named `input_video` for all incoming frames
|
||||
that will be provided by your device's camera.
|
||||
@@ -62,7 +82,7 @@ packets in the `luma_video` stream and outputs results in `output_video` output
|
||||
stream.
|
||||
|
||||
Our Android application will display the output image frames of the
|
||||
`sobel_video` stream.
|
||||
`output_video` stream.
|
||||
|
||||
## Initial minimal application setup
|
||||
|
||||
@@ -72,15 +92,15 @@ applications using `bazel`.
|
||||
|
||||
Create a new directory where you will create your Android application. For
|
||||
example, the complete code of this tutorial can be found at
|
||||
`mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu`.
|
||||
We will refer to this path as `$APPLICATION_PATH` throughout the codelab.
|
||||
`mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic`. We
|
||||
will refer to this path as `$APPLICATION_PATH` throughout the codelab.
|
||||
|
||||
Note that in the path to the application:
|
||||
|
||||
* The application is named `edgedetectiongpu`.
|
||||
* The application is named `helloworld`.
|
||||
* The `$PACKAGE_PATH` of the application is
|
||||
`com.google.mediapipe.apps.edgdetectiongpu`. This is used in code snippets in
|
||||
this tutorial, so please remember to use your own `$PACKAGE_PATH` when you
|
||||
`com.google.mediapipe.apps.basic`. This is used in code snippets in this
|
||||
tutorial, so please remember to use your own `$PACKAGE_PATH` when you
|
||||
copy/use the code snippets.
|
||||
|
||||
Add a file `activity_main.xml` to `$APPLICATION_PATH/res/layout`. This displays
|
||||
@@ -111,7 +131,7 @@ Add a simple `MainActivity.java` to `$APPLICATION_PATH` which loads the content
|
||||
of the `activity_main.xml` layout as shown below:
|
||||
|
||||
```
|
||||
package com.google.mediapipe.apps.edgedetectiongpu;
|
||||
package com.google.mediapipe.apps.basic;
|
||||
|
||||
import android.os.Bundle;
|
||||
import androidx.appcompat.app.AppCompatActivity;
|
||||
@@ -133,7 +153,7 @@ launches `MainActivity` on application start:
|
||||
```
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="com.google.mediapipe.apps.edgedetectiongpu">
|
||||
package="com.google.mediapipe.apps.basic">
|
||||
|
||||
<uses-sdk
|
||||
android:minSdkVersion="19"
|
||||
@@ -141,11 +161,11 @@ launches `MainActivity` on application start:
|
||||
|
||||
<application
|
||||
android:allowBackup="true"
|
||||
android:label="@string/app_name"
|
||||
android:label="${appName}"
|
||||
android:supportsRtl="true"
|
||||
android:theme="@style/AppTheme">
|
||||
<activity
|
||||
android:name=".MainActivity"
|
||||
android:name="${mainActivity}"
|
||||
android:exported="true"
|
||||
android:screenOrientation="portrait">
|
||||
<intent-filter>
|
||||
@@ -158,17 +178,8 @@ launches `MainActivity` on application start:
|
||||
</manifest>
|
||||
```
|
||||
|
||||
To get `@string/app_name`, we need to add a file `strings.xml` to
|
||||
`$APPLICATION_PATH/res/values/`:
|
||||
|
||||
```
|
||||
<resources>
|
||||
<string name="app_name" translatable="false">Edge Detection GPU</string>
|
||||
</resources>
|
||||
```
|
||||
|
||||
Also, in our application we are using a `Theme.AppCompat` theme in the app, so
|
||||
we need appropriate theme references. Add `colors.xml` to
|
||||
In our application we are using a `Theme.AppCompat` theme in the app, so we need
|
||||
appropriate theme references. Add `colors.xml` to
|
||||
`$APPLICATION_PATH/res/values/`:
|
||||
|
||||
```
|
||||
@@ -196,11 +207,13 @@ Add `styles.xml` to `$APPLICATION_PATH/res/values/`:
|
||||
</resources>
|
||||
```
|
||||
|
||||
To build the application, add a `BUILD` file to `$APPLICATION_PATH`:
|
||||
To build the application, add a `BUILD` file to `$APPLICATION_PATH`, and
|
||||
`${appName}` and `${mainActivity}` in the manifest will be replaced by strings
|
||||
specified in `BUILD` as shown below.
|
||||
|
||||
```
|
||||
android_library(
|
||||
name = "mediapipe_lib",
|
||||
name = "basic_lib",
|
||||
srcs = glob(["*.java"]),
|
||||
manifest = "AndroidManifest.xml",
|
||||
resource_files = glob(["res/**"]),
|
||||
@@ -211,40 +224,42 @@ android_library(
|
||||
)
|
||||
|
||||
android_binary(
|
||||
name = "edgedetectiongpu",
|
||||
aapt_version = "aapt2",
|
||||
name = "helloworld",
|
||||
manifest = "AndroidManifest.xml",
|
||||
manifest_values = {"applicationId": "com.google.mediapipe.apps.edgedetectiongpu"},
|
||||
manifest_values = {
|
||||
"applicationId": "com.google.mediapipe.apps.basic",
|
||||
"appName": "Hello World",
|
||||
"mainActivity": ".MainActivity",
|
||||
},
|
||||
multidex = "native",
|
||||
deps = [
|
||||
":mediapipe_lib",
|
||||
":basic_lib",
|
||||
],
|
||||
)
|
||||
|
||||
```
|
||||
|
||||
The `android_library` rule adds dependencies for `MainActivity`, resource files
|
||||
and `AndroidManifest.xml`.
|
||||
|
||||
The `android_binary` rule, uses the `mediapipe_lib` Android library generated to
|
||||
The `android_binary` rule, uses the `basic_lib` Android library generated to
|
||||
build a binary APK for installation on your Android device.
|
||||
|
||||
To build the app, use the following command:
|
||||
|
||||
```
|
||||
bazel build -c opt --config=android_arm64 $APPLICATION_PATH
|
||||
bazel build -c opt --config=android_arm64 $APPLICATION_PATH:helloworld
|
||||
```
|
||||
|
||||
Install the generated APK file using `adb install`. For example:
|
||||
|
||||
```
|
||||
adb install bazel-bin/$APPLICATION_PATH/edgedetectiongpu.apk
|
||||
adb install bazel-bin/$APPLICATION_PATH/helloworld.apk
|
||||
```
|
||||
|
||||
Open the application on your device. It should display a screen with the text
|
||||
`Hello World!`.
|
||||
|
||||
{width="300"}
|
||||

|
||||
|
||||
## Using the camera via `CameraX`
|
||||
|
||||
@@ -361,7 +376,7 @@ Add the following line in the `$APPLICATION_PATH/res/values/strings.xml` file:
|
||||
When the user doesn't grant camera permission, the screen will now look like
|
||||
this:
|
||||
|
||||
{width="300"}
|
||||

|
||||
|
||||
Now, we will add the [`SurfaceTexture`] and [`SurfaceView`] objects to
|
||||
`MainActivity`:
|
||||
@@ -430,22 +445,58 @@ visible so that we can start seeing frames from the `previewFrameTexture`.
|
||||
|
||||
However, before starting the camera, we need to decide which camera we want to
|
||||
use. [`CameraXPreviewHelper`] inherits from [`CameraHelper`] which provides two
|
||||
options, `FRONT` and `BACK`. We will use `BACK` camera for this application to
|
||||
perform edge detection on a live scene that we view from the camera.
|
||||
options, `FRONT` and `BACK`. We can pass in the decision from the `BUILD` file
|
||||
as metadata such that no code change is required to build a another version of
|
||||
the app using a different camera.
|
||||
|
||||
Add the following line to define `CAMERA_FACING` for our application,
|
||||
Assuming we want to use `BACK` camera to perform edge detection on a live scene
|
||||
that we view from the camera, add the metadata into `AndroidManifest.xml`:
|
||||
|
||||
```
|
||||
private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
|
||||
...
|
||||
<meta-data android:name="cameraFacingFront" android:value="${cameraFacingFront}"/>
|
||||
</application>
|
||||
</manifest>
|
||||
```
|
||||
|
||||
`CAMERA_FACING` is a static variable as we will use the same camera throughout
|
||||
the application from start to finish.
|
||||
and specify the selection in `BUILD` in the `helloworld` android binary rule
|
||||
with a new entry in `manifest_values`:
|
||||
|
||||
```
|
||||
manifest_values = {
|
||||
"applicationId": "com.google.mediapipe.apps.basic",
|
||||
"appName": "Hello World",
|
||||
"mainActivity": ".MainActivity",
|
||||
"cameraFacingFront": "False",
|
||||
},
|
||||
```
|
||||
|
||||
Now, in `MainActivity` to retrieve the metadata specified in `manifest_values`,
|
||||
add an [`ApplicationInfo`] object:
|
||||
|
||||
```
|
||||
private ApplicationInfo applicationInfo;
|
||||
```
|
||||
|
||||
In the `onCreate()` function, add:
|
||||
|
||||
```
|
||||
try {
|
||||
applicationInfo =
|
||||
getPackageManager().getApplicationInfo(getPackageName(), PackageManager.GET_META_DATA);
|
||||
} catch (NameNotFoundException e) {
|
||||
Log.e(TAG, "Cannot find application info: " + e);
|
||||
}
|
||||
```
|
||||
|
||||
Now add the following line at the end of the `startCamera()` function:
|
||||
|
||||
```
|
||||
cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
|
||||
CameraHelper.CameraFacing cameraFacing =
|
||||
applicationInfo.metaData.getBoolean("cameraFacingFront", false)
|
||||
? CameraHelper.CameraFacing.FRONT
|
||||
: CameraHelper.CameraFacing.BACK;
|
||||
cameraHelper.startCamera(this, cameraFacing, /*surfaceTexture=*/ null);
|
||||
```
|
||||
|
||||
At this point, the application should build successfully. However, when you run
|
||||
@@ -582,29 +633,18 @@ First, add dependencies to all calculator code in the `libmediapipe_jni.so`
|
||||
build rule:
|
||||
|
||||
```
|
||||
"//mediapipe/graphs/edge_detection:android_calculators",
|
||||
"//mediapipe/graphs/edge_detection:mobile_calculators",
|
||||
```
|
||||
|
||||
MediaPipe graphs are `.pbtxt` files, but to use them in the application, we need
|
||||
to use the `mediapipe_binary_graph` build rule to generate a `.binarypb` file.
|
||||
We can then use an application specific alias for the graph via the `genrule`
|
||||
build rule. Add the following `genrule` to use an alias for the edge detection
|
||||
graph:
|
||||
|
||||
```
|
||||
genrule(
|
||||
name = "binary_graph",
|
||||
srcs = ["//mediapipe/graphs/edge_detection:android_gpu_binary_graph"],
|
||||
outs = ["edgedetectiongpu.binarypb"],
|
||||
cmd = "cp $< $@",
|
||||
)
|
||||
```
|
||||
|
||||
Then in the `mediapipe_lib` build rule, add assets:
|
||||
In the `helloworld` android binary build rule, add the `mediapipe_binary_graph`
|
||||
target specific to the graph as an asset:
|
||||
|
||||
```
|
||||
assets = [
|
||||
":binary_graph",
|
||||
"//mediapipe/graphs/edge_detection:mobile_gpu_binary_graph",
|
||||
],
|
||||
assets_dir = "",
|
||||
```
|
||||
@@ -612,6 +652,26 @@ assets_dir = "",
|
||||
In the `assets` build rule, you can also add other assets such as TensorFlowLite
|
||||
models used in your graph.
|
||||
|
||||
In addition, add additional `manifest_values` for properties specific to the
|
||||
graph, to be later retrieved in `MainActivity`:
|
||||
|
||||
```
|
||||
manifest_values = {
|
||||
"applicationId": "com.google.mediapipe.apps.basic",
|
||||
"appName": "Hello World",
|
||||
"mainActivity": ".MainActivity",
|
||||
"cameraFacingFront": "False",
|
||||
"binaryGraphName": "mobile_gpu.binarypb",
|
||||
"inputVideoStreamName": "input_video",
|
||||
"outputVideoStreamName": "output_video",
|
||||
},
|
||||
```
|
||||
|
||||
Note that `binaryGraphName` indicates the filename of the binary graph,
|
||||
determined by the `output_name` field in the `mediapipe_binary_graph` target.
|
||||
`inputVideoStreamName` and `outputVideoStreamName` are the input and output
|
||||
video stream name specified in the graph respectively.
|
||||
|
||||
Now, the `MainActivity` needs to load the MediaPipe framework. Also, the
|
||||
framework uses OpenCV, so `MainActvity` should also load `OpenCV`. Use the
|
||||
following code in `MainActivity` (inside the class, but not inside any function)
|
||||
@@ -621,7 +681,7 @@ to load both dependencies:
|
||||
static {
|
||||
// Load all native libraries needed by the app.
|
||||
System.loadLibrary("mediapipe_jni");
|
||||
System.loadLibrary("opencv_java4");
|
||||
System.loadLibrary("opencv_java3");
|
||||
}
|
||||
```
|
||||
|
||||
@@ -635,20 +695,11 @@ Initialize the asset manager in `onCreate(Bundle)` before initializing
|
||||
`eglManager`:
|
||||
|
||||
```
|
||||
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
|
||||
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
|
||||
// binary graphs.
|
||||
AndroidAssetUtil.initializeNativeAssetManager(this);
|
||||
```
|
||||
|
||||
Declare a static variable with the graph name, the name of the input stream and
|
||||
the name of the output stream:
|
||||
|
||||
```
|
||||
private static final String BINARY_GRAPH_NAME = "edgedetectiongpu.binarypb";
|
||||
private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
|
||||
private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
|
||||
```
|
||||
|
||||
Now, we need to setup a [`FrameProcessor`] object that sends camera frames
|
||||
prepared by the `converter` to the MediaPipe graph and runs the graph, prepares
|
||||
the output and then updates the `previewDisplayView` to display the output. Add
|
||||
@@ -665,9 +716,9 @@ processor =
|
||||
new FrameProcessor(
|
||||
this,
|
||||
eglManager.getNativeContext(),
|
||||
BINARY_GRAPH_NAME,
|
||||
INPUT_VIDEO_STREAM_NAME,
|
||||
OUTPUT_VIDEO_STREAM_NAME);
|
||||
applicationInfo.metaData.getString("binaryGraphName"),
|
||||
applicationInfo.metaData.getString("inputVideoStreamName"),
|
||||
applicationInfo.metaData.getString("outputVideoStreamName"));
|
||||
```
|
||||
|
||||
The `processor` needs to consume the converted frames from the `converter` for
|
||||
@@ -701,26 +752,25 @@ And that's it! You should now be able to successfully build and run the
|
||||
application on the device and see Sobel edge detection running on a live camera
|
||||
feed! Congrats!
|
||||
|
||||
{width="300"}
|
||||

|
||||
|
||||
If you ran into any issues, please see the full code of the tutorial
|
||||
[here](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu).
|
||||
[here](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic).
|
||||
|
||||
[`ApplicationInfo`]:https://developer.android.com/reference/android/content/pm/ApplicationInfo
|
||||
[`AndroidAssetUtil`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/framework/AndroidAssetUtil.java
|
||||
[Bazel]:https://bazel.build/
|
||||
[`CameraHelper`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/CameraHelper.java
|
||||
[CameraX]:https://developer.android.com/training/camerax
|
||||
[`CameraXPreviewHelper`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/CameraXPreviewHelper.java
|
||||
[developer options]:https://developer.android.com/studio/debug/dev-options
|
||||
[`edge_detection_android_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_android_gpu.pbtxt
|
||||
[`EdgeDetectionGPU` example]:https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/
|
||||
[`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt
|
||||
[`EglManager`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/glutil/EglManager.java
|
||||
[`ExternalTextureConverter`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java
|
||||
[`FrameLayout`]:https://developer.android.com/reference/android/widget/FrameLayout
|
||||
[`FrameProcessor`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/FrameProcessor.java
|
||||
[MediaPipe installation guide]:./install.md
|
||||
[`PermissionHelper`]: https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/PermissionHelper.java
|
||||
[Setting up Android SDK and NDK]:./install.md#setting-up-android-sdk-and-ndk
|
||||
[`SurfaceHolder.Callback`]:https://developer.android.com/reference/android/view/SurfaceHolder.Callback.html
|
||||
[`SurfaceView`]:https://developer.android.com/reference/android/view/SurfaceView
|
||||
[`SurfaceView`]:https://developer.android.com/reference/android/view/SurfaceView
|
||||
@@ -1,4 +1,16 @@
|
||||
## Hello World for C++
|
||||
---
|
||||
layout: default
|
||||
title: Hello World! on Desktop (C++)
|
||||
parent: Getting Started
|
||||
nav_order: 5
|
||||
---
|
||||
|
||||
# Hello World! on Desktop (C++)
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
1. Ensure you have a working version of MediaPipe. See
|
||||
[installation instructions](./install.md).
|
||||
@@ -11,7 +23,7 @@
|
||||
|
||||
$ export GLOG_logtostderr=1
|
||||
# Need bazel flag 'MEDIAPIPE_DISABLE_GPU=1' as desktop GPU is not supported currently.
|
||||
$ bazel run --define 'MEDIAPIPE_DISABLE_GPU=1' \
|
||||
$ bazel run --define MEDIAPIPE_DISABLE_GPU=1 \
|
||||
mediapipe/examples/desktop/hello_world:hello_world
|
||||
|
||||
# It should print 10 rows of Hello World!
|
||||
@@ -52,7 +64,7 @@
|
||||
You can visualize this graph using
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev) by pasting the
|
||||
CalculatorGraphConfig content below into the visualizer. See
|
||||
[here](./visualizer.md) for help on the visualizer.
|
||||
[here](../tools/visualizer.md) for help on the visualizer.
|
||||
|
||||
```bash
|
||||
input_stream: "in"
|
||||
@@ -72,7 +84,7 @@
|
||||
This graph consists of 1 graph input stream (`in`) and 1 graph output stream
|
||||
(`out`), and 2 [`PassThroughCalculator`]s connected serially.
|
||||
|
||||
{width="200"}
|
||||

|
||||
|
||||
4. Before running the graph, an `OutputStreamPoller` object is connected to the
|
||||
output stream in order to later retrieve the graph output, and a graph run
|
||||
@@ -0,0 +1,560 @@
|
||||
---
|
||||
layout: default
|
||||
title: Hello World! on iOS
|
||||
parent: Getting Started
|
||||
nav_order: 4
|
||||
---
|
||||
|
||||
# Hello World! on iOS
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Introduction
|
||||
|
||||
This codelab uses MediaPipe on an iOS device.
|
||||
|
||||
### What you will learn
|
||||
|
||||
How to develop an iOS application that uses MediaPipe and run a MediaPipe
|
||||
graph on iOS.
|
||||
|
||||
### What you will build
|
||||
|
||||
A simple camera app for real-time Sobel edge detection applied to a live video
|
||||
stream on an iOS device.
|
||||
|
||||

|
||||
|
||||
## Setup
|
||||
|
||||
1. Install MediaPipe on your system, see [MediaPipe installation guide] for
|
||||
details.
|
||||
2. Setup your iOS device for development.
|
||||
3. Setup [Bazel] on your system to build and deploy the iOS app.
|
||||
|
||||
## Graph for edge detection
|
||||
|
||||
We will be using the following graph, [`edge_detection_mobile_gpu.pbtxt`]:
|
||||
|
||||
```
|
||||
# MediaPipe graph that performs GPU Sobel edge detection on a live video stream.
|
||||
# Used in the examples
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/edgedetectiongpu.
|
||||
# mediapipe/examples/ios/edgedetectiongpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
output_stream: "output_video"
|
||||
|
||||
# Converts RGB images into luminance images, still stored in RGB format.
|
||||
node: {
|
||||
calculator: "LuminanceCalculator"
|
||||
input_stream: "input_video"
|
||||
output_stream: "luma_video"
|
||||
}
|
||||
|
||||
# Applies the Sobel filter to luminance images sotred in RGB format.
|
||||
node: {
|
||||
calculator: "SobelEdgesCalculator"
|
||||
input_stream: "luma_video"
|
||||
output_stream: "output_video"
|
||||
}
|
||||
```
|
||||
|
||||
A visualization of the graph is shown below:
|
||||
|
||||

|
||||
|
||||
This graph has a single input stream named `input_video` for all incoming frames
|
||||
that will be provided by your device's camera.
|
||||
|
||||
The first node in the graph, `LuminanceCalculator`, takes a single packet (image
|
||||
frame) and applies a change in luminance using an OpenGL shader. The resulting
|
||||
image frame is sent to the `luma_video` output stream.
|
||||
|
||||
The second node, `SobelEdgesCalculator` applies edge detection to incoming
|
||||
packets in the `luma_video` stream and outputs results in `output_video` output
|
||||
stream.
|
||||
|
||||
Our iOS application will display the output image frames of the `output_video`
|
||||
stream.
|
||||
|
||||
## Initial minimal application setup
|
||||
|
||||
We first start with a simple iOS application and demonstrate how to use `bazel`
|
||||
to build it.
|
||||
|
||||
First, create an XCode project via File > New > Single View App.
|
||||
|
||||
Set the product name to "EdgeDetectionGpu", and use an appropriate organization
|
||||
identifier, such as `com.google.mediapipe`. The organization identifier
|
||||
alongwith the product name will be the `bundle_id` for the application, such as
|
||||
`com.google.mediapipe.EdgeDetectionGpu`.
|
||||
|
||||
Set the language to Objective-C.
|
||||
|
||||
Save the project to an appropriate location. Let's call this
|
||||
`$PROJECT_TEMPLATE_LOC`. So your project will be in the
|
||||
`$PROJECT_TEMPLATE_LOC/EdgeDetectionGpu` directory. This directory will contain
|
||||
another directory named `EdgeDetectionGpu` and an `EdgeDetectionGpu.xcodeproj` file.
|
||||
|
||||
The `EdgeDetectionGpu.xcodeproj` will not be useful for this tutorial, as we will
|
||||
use bazel to build the iOS application. The content of the
|
||||
`$PROJECT_TEMPLATE_LOC/EdgeDetectionGpu/EdgeDetectionGpu` directory is listed below:
|
||||
|
||||
1. `AppDelegate.h` and `AppDelegate.m`
|
||||
2. `ViewController.h` and `ViewController.m`
|
||||
3. `main.m`
|
||||
4. `Info.plist`
|
||||
5. `Main.storyboard` and `Launch.storyboard`
|
||||
6. `Assets.xcassets` directory.
|
||||
|
||||
Copy these files to a directory named `EdgeDetectionGpu` to a location that can
|
||||
access the MediaPipe source code. For example, the source code of the
|
||||
application that we will build in this tutorial is located in
|
||||
`mediapipe/examples/ios/EdgeDetectionGpu`. We will refer to this path as the
|
||||
`$APPLICATION_PATH` throughout the codelab.
|
||||
|
||||
Note: MediaPipe provides Objective-C bindings for iOS. The edge detection
|
||||
application in this tutorial and all iOS examples using MediaPipe use
|
||||
Objective-C with C++ in `.mm` files.
|
||||
|
||||
Create a `BUILD` file in the `$APPLICATION_PATH` and add the following build
|
||||
rules:
|
||||
|
||||
```
|
||||
MIN_IOS_VERSION = "10.0"
|
||||
|
||||
load(
|
||||
"@build_bazel_rules_apple//apple:ios.bzl",
|
||||
"ios_application",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "EdgeDetectionGpuApp",
|
||||
bundle_id = "com.google.mediapipe.EdgeDetectionGpu",
|
||||
families = [
|
||||
"iphone",
|
||||
"ipad",
|
||||
],
|
||||
infoplists = ["Info.plist"],
|
||||
minimum_os_version = MIN_IOS_VERSION,
|
||||
provisioning_profile = "//mediapipe/examples/ios:developer_provisioning_profile",
|
||||
deps = [":EdgeDetectionGpuAppLibrary"],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
name = "EdgeDetectionGpuAppLibrary",
|
||||
srcs = [
|
||||
"AppDelegate.m",
|
||||
"ViewController.m",
|
||||
"main.m",
|
||||
],
|
||||
hdrs = [
|
||||
"AppDelegate.h",
|
||||
"ViewController.h",
|
||||
],
|
||||
data = [
|
||||
"Base.lproj/LaunchScreen.storyboard",
|
||||
"Base.lproj/Main.storyboard",
|
||||
],
|
||||
sdk_frameworks = [
|
||||
"UIKit",
|
||||
],
|
||||
deps = [],
|
||||
)
|
||||
```
|
||||
|
||||
The `objc_library` rule adds dependencies for the `AppDelegate` and
|
||||
`ViewController` classes, `main.m` and the application storyboards. The
|
||||
templated app depends only on the `UIKit` SDK.
|
||||
|
||||
The `ios_application` rule uses the `EdgeDetectionGpuAppLibrary` Objective-C
|
||||
library generated to build an iOS application for installation on your iOS
|
||||
device.
|
||||
|
||||
Note: You need to point to your own iOS developer provisioning profile to be
|
||||
able to run the application on your iOS device.
|
||||
|
||||
To build the app, use the following command in a terminal:
|
||||
|
||||
```
|
||||
bazel build -c opt --config=ios_arm64 <$APPLICATION_PATH>:EdgeDetectionGpuApp'
|
||||
```
|
||||
|
||||
For example, to build the `EdgeDetectionGpuApp` application in
|
||||
`mediapipe/examples/ios/edgedetectiongpu`, use the following
|
||||
command:
|
||||
|
||||
```
|
||||
bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/edgedetectiongpu:EdgeDetectionGpuApp
|
||||
```
|
||||
|
||||
Then, go back to XCode, open Window > Devices and Simulators, select your
|
||||
device, and add the `.ipa` file generated by the command above to your device.
|
||||
Here is the document on [setting up and compiling](./building_examples.md#ios) iOS
|
||||
MediaPipe apps.
|
||||
|
||||
Open the application on your device. Since it is empty, it should display a
|
||||
blank white screen.
|
||||
|
||||
## Use the camera for the live view feed
|
||||
|
||||
In this tutorial, we will use the `MPPCameraInputSource` class to access and
|
||||
grab frames from the camera. This class uses the `AVCaptureSession` API to get
|
||||
the frames from the camera.
|
||||
|
||||
But before using this class, change the `Info.plist` file to support camera
|
||||
usage in the app.
|
||||
|
||||
In `ViewController.m`, add the following import line:
|
||||
|
||||
```
|
||||
#import "mediapipe/objc/MPPCameraInputSource.h"
|
||||
```
|
||||
|
||||
Add the following to its implementation block to create an object
|
||||
`_cameraSource`:
|
||||
|
||||
```
|
||||
@implementation ViewController {
|
||||
// Handles camera access via AVCaptureSession library.
|
||||
MPPCameraInputSource* _cameraSource;
|
||||
}
|
||||
```
|
||||
|
||||
Add the following code to `viewDidLoad()`:
|
||||
|
||||
```
|
||||
-(void)viewDidLoad {
|
||||
[super viewDidLoad];
|
||||
|
||||
_cameraSource = [[MPPCameraInputSource alloc] init];
|
||||
_cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
|
||||
_cameraSource.cameraPosition = AVCaptureDevicePositionBack;
|
||||
// The frame's native format is rotated with respect to the portrait orientation.
|
||||
_cameraSource.orientation = AVCaptureVideoOrientationPortrait;
|
||||
}
|
||||
```
|
||||
|
||||
The code initializes `_cameraSource`, sets the capture session preset, and which
|
||||
camera to use.
|
||||
|
||||
We need to get frames from the `_cameraSource` into our application
|
||||
`ViewController` to display them. `MPPCameraInputSource` is a subclass of
|
||||
`MPPInputSource`, which provides a protocol for its delegates, namely the
|
||||
`MPPInputSourceDelegate`. So our application `ViewController` can be a delegate
|
||||
of `_cameraSource`.
|
||||
|
||||
To handle camera setup and process incoming frames, we should use a queue
|
||||
different from the main queue. Add the following to the implementation block of
|
||||
the `ViewController`:
|
||||
|
||||
```
|
||||
// Process camera frames on this queue.
|
||||
dispatch_queue_t _videoQueue;
|
||||
```
|
||||
|
||||
In `viewDidLoad()`, add the following line after initializing the
|
||||
`_cameraSource` object:
|
||||
|
||||
```
|
||||
[_cameraSource setDelegate:self queue:_videoQueue];
|
||||
```
|
||||
|
||||
And add the following code to initialize the queue before setting up the
|
||||
`_cameraSource` object:
|
||||
|
||||
```
|
||||
dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class(
|
||||
DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0);
|
||||
_videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute);
|
||||
```
|
||||
|
||||
We will use a serial queue with the priority `QOS_CLASS_USER_INTERACTIVE` for
|
||||
processing camera frames.
|
||||
|
||||
Add the following line after the header imports at the top of the file, before
|
||||
the interface/implementation of the `ViewController`:
|
||||
|
||||
```
|
||||
static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
|
||||
```
|
||||
|
||||
Before implementing any method from `MPPInputSourceDelegate` protocol, we must
|
||||
first set up a way to display the camera frames. MediaPipe provides another
|
||||
utility called `MPPLayerRenderer` to display images on the screen. This utility
|
||||
can be used to display `CVPixelBufferRef` objects, which is the type of the
|
||||
images provided by `MPPCameraInputSource` to its delegates.
|
||||
|
||||
To display images of the screen, we need to add a new `UIView` object called
|
||||
`_liveView` to the `ViewController`.
|
||||
|
||||
Add the following lines to the implementation block of the `ViewController`:
|
||||
|
||||
```
|
||||
// Display the camera preview frames.
|
||||
IBOutlet UIView* _liveView;
|
||||
// Render frames in a layer.
|
||||
MPPLayerRenderer* _renderer;
|
||||
```
|
||||
|
||||
Go to `Main.storyboard`, add a `UIView` object from the object library to the
|
||||
`View` of the `ViewController` class. Add a referencing outlet from this view to
|
||||
the `_liveView` object you just added to the `ViewController` class. Resize the
|
||||
view so that it is centered and covers the entire application screen.
|
||||
|
||||
Go back to `ViewController.m` and add the following code to `viewDidLoad()` to
|
||||
initialize the `_renderer` object:
|
||||
|
||||
```
|
||||
_renderer = [[MPPLayerRenderer alloc] init];
|
||||
_renderer.layer.frame = _liveView.layer.bounds;
|
||||
[_liveView.layer addSublayer:_renderer.layer];
|
||||
_renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop;
|
||||
```
|
||||
|
||||
To get frames from the camera, we will implement the following method:
|
||||
|
||||
```
|
||||
// Must be invoked on _videoQueue.
|
||||
- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
|
||||
timestamp:(CMTime)timestamp
|
||||
fromSource:(MPPInputSource*)source {
|
||||
if (source != _cameraSource) {
|
||||
NSLog(@"Unknown source: %@", source);
|
||||
return;
|
||||
}
|
||||
// Display the captured image on the screen.
|
||||
CFRetain(imageBuffer);
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[_renderer renderPixelBuffer:imageBuffer];
|
||||
CFRelease(imageBuffer);
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
This is a delegate method of `MPPInputSource`. We first check that we are
|
||||
getting frames from the right source, i.e. the `_cameraSource`. Then we display
|
||||
the frame received from the camera via `_renderer` on the main queue.
|
||||
|
||||
Now, we need to start the camera as soon as the view to display the frames is
|
||||
about to appear. To do this, we will implement the
|
||||
`viewWillAppear:(BOOL)animated` function:
|
||||
|
||||
```
|
||||
-(void)viewWillAppear:(BOOL)animated {
|
||||
[super viewWillAppear:animated];
|
||||
}
|
||||
```
|
||||
|
||||
Before we start running the camera, we need the user's permission to access it.
|
||||
`MPPCameraInputSource` provides a function
|
||||
`requestCameraAccessWithCompletionHandler:(void (^_Nullable)(BOOL
|
||||
granted))handler` to request camera access and do some work when the user has
|
||||
responded. Add the following code to `viewWillAppear:animated`:
|
||||
|
||||
```
|
||||
[_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
|
||||
if (granted) {
|
||||
dispatch_async(_videoQueue, ^{
|
||||
[_cameraSource start];
|
||||
});
|
||||
}
|
||||
}];
|
||||
```
|
||||
|
||||
Before building the application, add the following dependencies to your `BUILD`
|
||||
file:
|
||||
|
||||
```
|
||||
sdk_frameworks = [
|
||||
"AVFoundation",
|
||||
"CoreGraphics",
|
||||
"CoreMedia",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/objc:mediapipe_framework_ios",
|
||||
"//mediapipe/objc:mediapipe_input_sources_ios",
|
||||
"//mediapipe/objc:mediapipe_layer_renderer",
|
||||
],
|
||||
```
|
||||
|
||||
Now build and run the application on your iOS device. You should see a live
|
||||
camera view feed after accepting camera permissions.
|
||||
|
||||
We are now ready to use camera frames in a MediaPipe graph.
|
||||
|
||||
## Using a MediaPipe graph in iOS
|
||||
|
||||
### Add relevant dependencies
|
||||
|
||||
We already added the dependencies of the MediaPipe framework code which contains
|
||||
the iOS API to use a MediaPipe graph. To use a MediaPipe graph, we need to add a
|
||||
dependency on the graph we intend to use in our application. Add the following
|
||||
line to the `data` list in your `BUILD` file:
|
||||
|
||||
```
|
||||
"//mediapipe/graphs/edge_detection:mobile_gpu_binary_graph",
|
||||
```
|
||||
|
||||
Now add the dependency to the calculators used in this graph in the `deps` field
|
||||
in the `BUILD` file:
|
||||
|
||||
```
|
||||
"//mediapipe/graphs/edge_detection:mobile_calculators",
|
||||
```
|
||||
|
||||
Finally, rename the file `ViewController.m` to `ViewController.mm` to support
|
||||
Objective-C++.
|
||||
|
||||
### Use the graph in `ViewController`
|
||||
|
||||
Declare a static constant with the name of the graph, the input stream and the
|
||||
output stream:
|
||||
|
||||
```
|
||||
static NSString* const kGraphName = @"mobile_gpu";
|
||||
|
||||
static const char* kInputStream = "input_video";
|
||||
static const char* kOutputStream = "output_video";
|
||||
```
|
||||
|
||||
Add the following property to the interface of the `ViewController`:
|
||||
|
||||
```
|
||||
// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and
|
||||
// sent video frames on _videoQueue.
|
||||
@property(nonatomic) MPPGraph* mediapipeGraph;
|
||||
```
|
||||
|
||||
As explained in the comment above, we will initialize this graph in
|
||||
`viewDidLoad` first. To do so, we need to load the graph from the `.pbtxt` file
|
||||
using the following function:
|
||||
|
||||
```
|
||||
+ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
|
||||
// Load the graph config resource.
|
||||
NSError* configLoadError = nil;
|
||||
NSBundle* bundle = [NSBundle bundleForClass:[self class]];
|
||||
if (!resource || resource.length == 0) {
|
||||
return nil;
|
||||
}
|
||||
NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
|
||||
NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
|
||||
if (!data) {
|
||||
NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError);
|
||||
return nil;
|
||||
}
|
||||
|
||||
// Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object.
|
||||
mediapipe::CalculatorGraphConfig config;
|
||||
config.ParseFromArray(data.bytes, data.length);
|
||||
|
||||
// Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
|
||||
MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
|
||||
[newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
|
||||
return newGraph;
|
||||
}
|
||||
```
|
||||
|
||||
Use this function to initialize the graph in `viewDidLoad` as follows:
|
||||
|
||||
```
|
||||
self.mediapipeGraph = [[self class] loadGraphFromResource:kGraphName];
|
||||
```
|
||||
|
||||
The graph should send the results of processing camera frames back to the
|
||||
`ViewController`. Add the following line after initializing the graph to set the
|
||||
`ViewController` as a delegate of the `mediapipeGraph` object:
|
||||
|
||||
```
|
||||
self.mediapipeGraph.delegate = self;
|
||||
```
|
||||
|
||||
To avoid memory contention while processing frames from the live video feed, add
|
||||
the following line:
|
||||
|
||||
```
|
||||
// Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
|
||||
self.mediapipeGraph.maxFramesInFlight = 2;
|
||||
```
|
||||
|
||||
Now, start the graph when the user has granted the permission to use the camera
|
||||
in our app:
|
||||
|
||||
```
|
||||
[_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
|
||||
if (granted) {
|
||||
// Start running self.mediapipeGraph.
|
||||
NSError* error;
|
||||
if (![self.mediapipeGraph startWithError:&error]) {
|
||||
NSLog(@"Failed to start graph: %@", error);
|
||||
}
|
||||
|
||||
dispatch_async(_videoQueue, ^{
|
||||
[_cameraSource start];
|
||||
});
|
||||
}
|
||||
}];
|
||||
```
|
||||
|
||||
Note: It is important to start the graph before starting the camera, so that
|
||||
the graph is ready to process frames as soon as the camera starts sending them.
|
||||
|
||||
Earlier, when we received frames from the camera in the `processVideoFrame`
|
||||
function, we displayed them in the `_liveView` using the `_renderer`. Now, we
|
||||
need to send those frames to the graph and render the results instead. Modify
|
||||
this function's implementation to do the following:
|
||||
|
||||
```
|
||||
- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
|
||||
timestamp:(CMTime)timestamp
|
||||
fromSource:(MPPInputSource*)source {
|
||||
if (source != _cameraSource) {
|
||||
NSLog(@"Unknown source: %@", source);
|
||||
return;
|
||||
}
|
||||
[self.mediapipeGraph sendPixelBuffer:imageBuffer
|
||||
intoStream:kInputStream
|
||||
packetType:MPPPacketTypePixelBuffer];
|
||||
}
|
||||
```
|
||||
|
||||
We send the `imageBuffer` to `self.mediapipeGraph` as a packet of type
|
||||
`MPPPacketTypePixelBuffer` into the input stream `kInputStream`, i.e.
|
||||
"input_video".
|
||||
|
||||
The graph will run with this input packet and output a result in
|
||||
`kOutputStream`, i.e. "output_video". We can implement the following delegate
|
||||
method to receive packets on this output stream and display them on the screen:
|
||||
|
||||
```
|
||||
- (void)mediapipeGraph:(MPPGraph*)graph
|
||||
didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
fromStream:(const std::string&)streamName {
|
||||
if (streamName == kOutputStream) {
|
||||
// Display the captured image on the screen.
|
||||
CVPixelBufferRetain(pixelBuffer);
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[_renderer renderPixelBuffer:pixelBuffer];
|
||||
CVPixelBufferRelease(pixelBuffer);
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
And that is all! Build and run the app on your iOS device. You should see the
|
||||
results of running the edge detection graph on a live video feed. Congrats!
|
||||
|
||||

|
||||
|
||||
If you ran into any issues, please see the full code of the tutorial
|
||||
[here](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/edgedetectiongpu).
|
||||
|
||||
[Bazel]:https://bazel.build/
|
||||
[`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt
|
||||
[MediaPipe installation guide]:./install.md
|
||||
@@ -0,0 +1,48 @@
|
||||
---
|
||||
layout: default
|
||||
title: Getting Help
|
||||
parent: Getting Started
|
||||
nav_order: 8
|
||||
---
|
||||
|
||||
# Getting Help
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Technical questions
|
||||
|
||||
For help with technical or algorithmic questions, visit
|
||||
[Stack Overflow](https://stackoverflow.com/questions/tagged/mediapipe) to find
|
||||
answers and support from the MediaPipe community.
|
||||
|
||||
## Bugs and feature requests
|
||||
|
||||
To report bugs or make feature requests,
|
||||
[file an issue on GitHub](https://github.com/google/mediapipe/issues).
|
||||
|
||||
If you open a GitHub issue, here is our policy:
|
||||
|
||||
1. It must be a bug, a feature request, or a significant problem with documentation (for small doc fixes please send a PR instead).
|
||||
2. The form below must be filled out.
|
||||
|
||||
**Here's why we have that policy**: MediaPipe developers respond to issues. We want to focus on work that benefits the whole community, e.g., fixing bugs and adding features. Support only helps individuals. GitHub also notifies thousands of people when issues are filed. We want them to see you communicating an interesting problem, rather than being redirected to Stack Overflow.
|
||||
|
||||
------------------------
|
||||
|
||||
### System information
|
||||
- **Have I written custom code**:
|
||||
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
|
||||
- **Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device**:
|
||||
- **Bazel version**:
|
||||
- **Android Studio, NDK, SDK versions (if issue is related to building in mobile dev enviroment)**:
|
||||
- **Xcode & Tulsi version (if issue is related to building in mobile dev enviroment)**:
|
||||
- **Exact steps to reproduce**:
|
||||
|
||||
### Describe the problem
|
||||
Describe the problem clearly here. Be sure to convey here why it's a bug in MediaPipe or a feature request.
|
||||
|
||||
### Source code / logs
|
||||
Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached instead of being pasted into the issue as text.
|
||||
@@ -0,0 +1,697 @@
|
||||
---
|
||||
layout: default
|
||||
title: Installation
|
||||
parent: Getting Started
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
# Installation
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
Note: To interoperate with OpenCV, OpenCV 3.x and above are preferred. OpenCV
|
||||
2.x currently works but interoperability support may be deprecated in the
|
||||
future.
|
||||
|
||||
Note: If you plan to use TensorFlow calculators and example apps, there is a
|
||||
known issue with gcc and g++ version 6.3 and 7.3. Please use other versions.
|
||||
|
||||
Note: To make Mediapipe work with TensorFlow, please set Python 3.7 as the
|
||||
default Python version and install the Python "six" library by running `pip3
|
||||
install --user six`.
|
||||
|
||||
Note: To build and run Android example apps, see these
|
||||
[instructions](./building_examples.md#android). To build and run iOS example
|
||||
apps, see these [instructions](./building_examples.md#ios).
|
||||
|
||||
## Installing on Debian and Ubuntu
|
||||
|
||||
1. Checkout MediaPipe repository.
|
||||
|
||||
```bash
|
||||
$ git clone https://github.com/google/mediapipe.git
|
||||
|
||||
# Change directory into MediaPipe root directory
|
||||
$ cd mediapipe
|
||||
```
|
||||
|
||||
2. Install Bazel.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
|
||||
to install Bazel 2.0 or higher.
|
||||
|
||||
For Nvidia Jetson and Raspberry Pi devices with ARM Ubuntu, Bazel needs to
|
||||
be built from source.
|
||||
|
||||
```bash
|
||||
# For Bazel 3.0.0
|
||||
wget https://github.com/bazelbuild/bazel/releases/download/3.0.0/bazel-3.0.0-dist.zip
|
||||
sudo apt-get install build-essential openjdk-8-jdk python zip unzip
|
||||
unzip bazel-3.0.0-dist.zip
|
||||
env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
|
||||
sudo cp output/bazel /usr/local/bin/
|
||||
```
|
||||
|
||||
3. Install OpenCV and FFmpeg.
|
||||
|
||||
Option 1. Use package manager tool to install the pre-compiled OpenCV
|
||||
libraries. FFmpeg will be installed via libopencv-video-dev.
|
||||
|
||||
Note: Debian 9 and Ubuntu 16.04 provide OpenCV 2.4.9. You may want to take
|
||||
option 2 or 3 to install OpenCV 3 or above.
|
||||
|
||||
```bash
|
||||
$ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
|
||||
libopencv-calib3d-dev libopencv-features2d-dev \
|
||||
libopencv-imgproc-dev libopencv-video-dev
|
||||
```
|
||||
|
||||
[`opencv_linux.BUILD`] is configured for x86_64 by default. For Nvidia
|
||||
Jetson and Raspberry Pi devices with ARM Ubuntu, the lib paths need to be
|
||||
modified.
|
||||
|
||||
```bash
|
||||
sed -i "s/x86_64-linux-gnu/aarch64-linux-gnu/g" third_party/opencv_linux.BUILD
|
||||
```
|
||||
|
||||
Option 2. Run [`setup_opencv.sh`] to automatically build OpenCV from source
|
||||
and modify MediaPipe's OpenCV config.
|
||||
|
||||
Option 3. Follow OpenCV's
|
||||
[documentation](https://docs.opencv.org/3.4.6/d7/d9f/tutorial_linux_install.html)
|
||||
to manually build OpenCV from source code.
|
||||
|
||||
Note: You may need to modify [`WORKSPACE`] and [`opencv_linux.BUILD`] to
|
||||
point MediaPipe to your own OpenCV libraries, e.g., if OpenCV 4 is installed
|
||||
in "/usr/local/", you need to update the "linux_opencv" new_local_repository
|
||||
rule in [`WORKSPACE`] and "opencv" cc_library rule in [`opencv_linux.BUILD`]
|
||||
like the following:
|
||||
|
||||
```bash
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr/local",
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "opencv",
|
||||
srcs = glob(
|
||||
[
|
||||
"lib/libopencv_core.so",
|
||||
"lib/libopencv_highgui.so",
|
||||
"lib/libopencv_imgcodecs.so",
|
||||
"lib/libopencv_imgproc.so",
|
||||
"lib/libopencv_video.so",
|
||||
"lib/libopencv_videoio.so",
|
||||
],
|
||||
),
|
||||
hdrs = glob(["include/opencv4/**/*.h*"]),
|
||||
includes = ["include/opencv4/"],
|
||||
linkstatic = 1,
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
```
|
||||
|
||||
4. For running desktop examples on Linux only (not on OS X) with GPU
|
||||
acceleration.
|
||||
|
||||
```bash
|
||||
# Requires a GPU with EGL driver support.
|
||||
# Can use mesa GPU libraries for desktop, (or Nvidia/AMD equivalent).
|
||||
sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
|
||||
|
||||
# To compile with GPU support, replace
|
||||
--define MEDIAPIPE_DISABLE_GPU=1
|
||||
# with
|
||||
--copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11
|
||||
# when building GPU examples.
|
||||
```
|
||||
|
||||
5. Run the [Hello World desktop example](./hello_world_desktop.md).
|
||||
|
||||
```bash
|
||||
$ export GLOG_logtostderr=1
|
||||
|
||||
# if you are running on Linux desktop with CPU only
|
||||
$ bazel run --define MEDIAPIPE_DISABLE_GPU=1 \
|
||||
mediapipe/examples/desktop/hello_world:hello_world
|
||||
|
||||
# If you are running on Linux desktop with GPU support enabled (via mesa drivers)
|
||||
$ bazel run --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
|
||||
mediapipe/examples/desktop/hello_world:hello_world
|
||||
|
||||
# Should print:
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
```
|
||||
|
||||
## Installing on CentOS
|
||||
|
||||
**Disclaimer**: Running MediaPipe on CentOS is experimental.
|
||||
|
||||
1. Checkout MediaPipe repository.
|
||||
|
||||
```bash
|
||||
$ git clone https://github.com/google/mediapipe.git
|
||||
|
||||
# Change directory into MediaPipe root directory
|
||||
$ cd mediapipe
|
||||
```
|
||||
|
||||
2. Install Bazel.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
|
||||
to install Bazel 2.0 or higher.
|
||||
|
||||
3. Install OpenCV.
|
||||
|
||||
Option 1. Use package manager tool to install the pre-compiled version.
|
||||
|
||||
Note: yum installs OpenCV 2.4.5, which may have an opencv/gstreamer
|
||||
[issue](https://github.com/opencv/opencv/issues/4592).
|
||||
|
||||
```bash
|
||||
$ sudo yum install opencv-devel
|
||||
```
|
||||
|
||||
Option 2. Build OpenCV from source code.
|
||||
|
||||
Note: You may need to modify [`WORKSPACE`] and [`opencv_linux.BUILD`] to
|
||||
point MediaPipe to your own OpenCV libraries, e.g., if OpenCV 4 is installed
|
||||
in "/usr/local/", you need to update the "linux_opencv" new_local_repository
|
||||
rule in [`WORKSPACE`] and "opencv" cc_library rule in [`opencv_linux.BUILD`]
|
||||
like the following:
|
||||
|
||||
```bash
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr/local",
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "opencv",
|
||||
srcs = glob(
|
||||
[
|
||||
"lib/libopencv_core.so",
|
||||
"lib/libopencv_highgui.so",
|
||||
"lib/libopencv_imgcodecs.so",
|
||||
"lib/libopencv_imgproc.so",
|
||||
"lib/libopencv_video.so",
|
||||
"lib/libopencv_videoio.so",
|
||||
],
|
||||
),
|
||||
hdrs = glob(["include/opencv4/**/*.h*"]),
|
||||
includes = ["include/opencv4/"],
|
||||
linkstatic = 1,
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
```
|
||||
|
||||
4. Run the [Hello World desktop example](./hello_world_desktop.md).
|
||||
|
||||
```bash
|
||||
$ export GLOG_logtostderr=1
|
||||
# Need bazel flag 'MEDIAPIPE_DISABLE_GPU=1' if you are running on Linux desktop with CPU only
|
||||
$ bazel run --define MEDIAPIPE_DISABLE_GPU=1 \
|
||||
mediapipe/examples/desktop/hello_world:hello_world
|
||||
|
||||
# Should print:
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
```
|
||||
|
||||
## Installing on macOS
|
||||
|
||||
1. Prework:
|
||||
|
||||
* Install [Homebrew](https://brew.sh).
|
||||
* Install [Xcode](https://developer.apple.com/xcode/) and its Command Line
|
||||
Tools by `xcode-select --install`.
|
||||
|
||||
2. Checkout MediaPipe repository.
|
||||
|
||||
```bash
|
||||
$ git clone https://github.com/google/mediapipe.git
|
||||
|
||||
$ cd mediapipe
|
||||
```
|
||||
|
||||
3. Install Bazel.
|
||||
|
||||
Option 1. Use package manager tool to install Bazel
|
||||
|
||||
```bash
|
||||
$ brew install bazel
|
||||
# Run 'bazel version' to check version of bazel
|
||||
```
|
||||
|
||||
Option 2. Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
|
||||
to install Bazel 2.0 or higher.
|
||||
|
||||
4. Install OpenCV and FFmpeg.
|
||||
|
||||
Option 1. Use HomeBrew package manager tool to install the pre-compiled
|
||||
OpenCV 3.4.5 libraries. FFmpeg will be installed via OpenCV.
|
||||
|
||||
```bash
|
||||
$ brew install opencv@3
|
||||
|
||||
# There is a known issue caused by the glog dependency. Uninstall glog.
|
||||
$ brew uninstall --ignore-dependencies glog
|
||||
```
|
||||
|
||||
Option 2. Use MacPorts package manager tool to install the OpenCV libraries.
|
||||
|
||||
```bash
|
||||
$ port install opencv
|
||||
```
|
||||
|
||||
Note: when using MacPorts, please edit the [`WORKSPACE`],
|
||||
[`opencv_macos.BUILD`], and [`ffmpeg_macos.BUILD`] files like the following:
|
||||
|
||||
```bash
|
||||
new_local_repository(
|
||||
name = "macos_opencv",
|
||||
build_file = "@//third_party:opencv_macos.BUILD",
|
||||
path = "/opt",
|
||||
)
|
||||
|
||||
new_local_repository(
|
||||
name = "macos_ffmpeg",
|
||||
build_file = "@//third_party:ffmpeg_macos.BUILD",
|
||||
path = "/opt",
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "opencv",
|
||||
srcs = glob(
|
||||
[
|
||||
"local/lib/libopencv_core.dylib",
|
||||
"local/lib/libopencv_highgui.dylib",
|
||||
"local/lib/libopencv_imgcodecs.dylib",
|
||||
"local/lib/libopencv_imgproc.dylib",
|
||||
"local/lib/libopencv_video.dylib",
|
||||
"local/lib/libopencv_videoio.dylib",
|
||||
],
|
||||
),
|
||||
hdrs = glob(["local/include/opencv2/**/*.h*"]),
|
||||
includes = ["local/include/"],
|
||||
linkstatic = 1,
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "libffmpeg",
|
||||
srcs = glob(
|
||||
[
|
||||
"local/lib/libav*.dylib",
|
||||
],
|
||||
),
|
||||
hdrs = glob(["local/include/libav*/*.h"]),
|
||||
includes = ["local/include/"],
|
||||
linkopts = [
|
||||
"-lavcodec",
|
||||
"-lavformat",
|
||||
"-lavutil",
|
||||
],
|
||||
linkstatic = 1,
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
```
|
||||
|
||||
5. Make sure that Python 3 and the Python "six" library are installed.
|
||||
|
||||
```
|
||||
$ brew install python
|
||||
$ sudo ln -s -f /usr/local/bin/python3.7 /usr/local/bin/python
|
||||
$ python --version
|
||||
Python 3.7.4
|
||||
$ pip3 install --user six
|
||||
```
|
||||
|
||||
6. Run the [Hello World desktop example](./hello_world_desktop.md).
|
||||
|
||||
```bash
|
||||
$ export GLOG_logtostderr=1
|
||||
# Need bazel flag 'MEDIAPIPE_DISABLE_GPU=1' as desktop GPU is currently not supported
|
||||
$ bazel run --define MEDIAPIPE_DISABLE_GPU=1 \
|
||||
mediapipe/examples/desktop/hello_world:hello_world
|
||||
|
||||
# Should print:
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
```
|
||||
|
||||
## Installing on Windows
|
||||
|
||||
**Disclaimer**: Running MediaPipe on Windows is experimental.
|
||||
|
||||
Note: building MediaPipe Android apps is still not possible on native
|
||||
Windows. Please do this in WSL instead and see the WSL setup instruction in the
|
||||
next section.
|
||||
|
||||
1. Install [MSYS2](https://www.msys2.org/) and edit the `%PATH%` environment
|
||||
variable.
|
||||
|
||||
If MSYS2 is installed to `C:\msys64`, add `C:\msys64\usr\bin` to your
|
||||
`%PATH%` environment variable.
|
||||
|
||||
2. Install necessary packages.
|
||||
|
||||
```
|
||||
C:\> pacman -S git patch unzip
|
||||
```
|
||||
|
||||
3. Install Python and allow the executable to edit the `%PATH%` environment
|
||||
variable.
|
||||
|
||||
Download Python Windows executable from
|
||||
https://www.python.org/downloads/windows/ and install.
|
||||
|
||||
4. Install Visual C++ Build Tools 2019 and WinSDK
|
||||
|
||||
Go to https://visualstudio.microsoft.com/visual-cpp-build-tools, download
|
||||
build tools, and install Microsoft Visual C++ 2019 Redistributable and
|
||||
Microsoft Build Tools 2019.
|
||||
|
||||
Download the WinSDK from
|
||||
https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/ and
|
||||
install.
|
||||
|
||||
5. Install Bazel and add the location of the Bazel executable to the `%PATH%`
|
||||
environment variable.
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
|
||||
to install Bazel 2.0 or higher.
|
||||
|
||||
6. Set Bazel variables.
|
||||
|
||||
```
|
||||
# Find the exact paths and version numbers from your local version.
|
||||
C:\> set BAZEL_VS=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools
|
||||
C:\> set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC
|
||||
C:\> set BAZEL_VC_FULL_VERSION=14.25.28610
|
||||
C:\> set BAZEL_WINSDK_FULL_VERSION=10.1.18362.1
|
||||
```
|
||||
|
||||
7. Checkout MediaPipe repository.
|
||||
|
||||
```
|
||||
C:\Users\Username\mediapipe_repo> git clone https://github.com/google/mediapipe.git
|
||||
|
||||
# Change directory into MediaPipe root directory
|
||||
C:\Users\Username\mediapipe_repo> cd mediapipe
|
||||
```
|
||||
|
||||
8. Install OpenCV.
|
||||
|
||||
Download the Windows executable from https://opencv.org/releases/ and
|
||||
install. We currently use OpenCV 3.4.10. Remember to edit the [`WORKSPACE`]
|
||||
file if OpenCV is not installed at `C:\opencv`.
|
||||
|
||||
```
|
||||
new_local_repository(
|
||||
name = "windows_opencv",
|
||||
build_file = "@//third_party:opencv_windows.BUILD",
|
||||
path = "C:\\<path to opencv>\\build",
|
||||
)
|
||||
```
|
||||
|
||||
9. Run the [Hello World desktop example](./hello_world_desktop.md).
|
||||
|
||||
Note: For building MediaPipe on Windows, please add `--action_env
|
||||
PYTHON_BIN_PATH="C:/path/to/python.exe"` to the build command.
|
||||
Alternatively, you can follow
|
||||
[issue 724](https://github.com/google/mediapipe/issues/724) to fix the
|
||||
python configuration manually.
|
||||
|
||||
```
|
||||
C:\Users\Username\mediapipe_repo>bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 --action_env PYTHON_BIN_PATH="C:/python_36/python.exe" mediapipe/examples/desktop/hello_world
|
||||
|
||||
C:\Users\Username\mediapipe_repo>set GLOG_logtostderr=1
|
||||
|
||||
C:\Users\Username\mediapipe_repo>bazel-bin\mediapipe\examples\desktop\hello_world\hello_world.exe
|
||||
|
||||
# should print:
|
||||
# I20200514 20:43:12.277598 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.278597 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.279618 1200 hello_world.cc:56] Hello World!
|
||||
# I20200514 20:43:12.280613 1200 hello_world.cc:56] Hello World!
|
||||
|
||||
```
|
||||
|
||||
## Installing on Windows Subsystem for Linux (WSL)
|
||||
|
||||
Note: The pre-built OpenCV packages don't support cameras in WSL. Unless you
|
||||
[compile](https://funvision.blogspot.com/2019/12/opencv-web-camera-and-video-streams-in.html)
|
||||
OpenCV with FFMPEG and GStreamer in WSL, the live demos won't work with any
|
||||
cameras. Alternatively, you use a video file as input.
|
||||
|
||||
1. Follow the
|
||||
[instruction](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to
|
||||
install Windows Sysystem for Linux (Ubuntu).
|
||||
|
||||
2. Install Windows ADB and start the ADB server in Windows.
|
||||
|
||||
Note: Windows' and WSL’s adb versions must be the same version, e.g., if WSL
|
||||
has ADB 1.0.39, you need to download the corresponding Windows ADB from
|
||||
[here](https://dl.google.com/android/repository/platform-tools_r26.0.1-windows.zip).
|
||||
|
||||
3. Launch WSL.
|
||||
|
||||
Note: All the following steps will be executed in WSL. The Windows directory
|
||||
of the Linux Subsystem can be found in
|
||||
C:\Users\YourUsername\AppData\Local\Packages\CanonicalGroupLimited.UbuntuonWindows_SomeID\LocalState\rootfs\home
|
||||
|
||||
4. Install the needed packages.
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
|
||||
```
|
||||
|
||||
5. Install Bazel.
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
|
||||
https://storage.googleapis.com/bazel/3.0.0/release/bazel-3.0.0-installer-linux-x86_64.sh && \
|
||||
sudo mkdir -p /usr/local/bazel/3.0.0 && \
|
||||
chmod 755 bazel-3.0.0-installer-linux-x86_64.sh && \
|
||||
sudo ./bazel-3.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.0.0 && \
|
||||
source /usr/local/bazel/3.0.0/lib/bazel/bin/bazel-complete.bash
|
||||
|
||||
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.0.0/lib/bazel/bin/bazel version && \
|
||||
alias bazel='/usr/local/bazel/3.0.0/lib/bazel/bin/bazel'
|
||||
```
|
||||
|
||||
6. Checkout MediaPipe repository.
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~$ git clone https://github.com/google/mediapipe.git
|
||||
|
||||
username@DESKTOP-TMVLBJ1:~$ cd mediapipe
|
||||
```
|
||||
|
||||
7. Install OpenCV and FFmpeg.
|
||||
|
||||
Option 1. Use package manager tool to install the pre-compiled OpenCV
|
||||
libraries. FFmpeg will be installed via libopencv-video-dev.
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~/mediapipe$ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
|
||||
libopencv-calib3d-dev libopencv-features2d-dev \
|
||||
libopencv-imgproc-dev libopencv-video-dev
|
||||
```
|
||||
|
||||
Option 2. Run [`setup_opencv.sh`] to automatically build OpenCV from source
|
||||
and modify MediaPipe's OpenCV config.
|
||||
|
||||
Option 3. Follow OpenCV's
|
||||
[documentation](https://docs.opencv.org/3.4.6/d7/d9f/tutorial_linux_install.html)
|
||||
to manually build OpenCV from source code.
|
||||
|
||||
Note: You may need to modify [`WORKSPACE`] and [`opencv_linux.BUILD`] to
|
||||
point MediaPipe to your own OpenCV libraries, e.g., if OpenCV 4 is installed
|
||||
in "/usr/local/", you need to update the "linux_opencv" new_local_repository
|
||||
rule in [`WORKSPACE`] and "opencv" cc_library rule in [`opencv_linux.BUILD`]
|
||||
like the following:
|
||||
|
||||
```bash
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
build_file = "@//third_party:opencv_linux.BUILD",
|
||||
path = "/usr/local",
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "opencv",
|
||||
srcs = glob(
|
||||
[
|
||||
"lib/libopencv_core.so",
|
||||
"lib/libopencv_highgui.so",
|
||||
"lib/libopencv_imgcodecs.so",
|
||||
"lib/libopencv_imgproc.so",
|
||||
"lib/libopencv_video.so",
|
||||
"lib/libopencv_videoio.so",
|
||||
],
|
||||
),
|
||||
hdrs = glob(["include/opencv4/**/*.h*"]),
|
||||
includes = ["include/opencv4/"],
|
||||
linkstatic = 1,
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
```
|
||||
|
||||
8. Run the [Hello World desktop example](./hello_world_desktop.md).
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~/mediapipe$ export GLOG_logtostderr=1
|
||||
|
||||
# Need bazel flag 'MEDIAPIPE_DISABLE_GPU=1' as desktop GPU is currently not supported
|
||||
username@DESKTOP-TMVLBJ1:~/mediapipe$ bazel run --define MEDIAPIPE_DISABLE_GPU=1 \
|
||||
mediapipe/examples/desktop/hello_world:hello_world
|
||||
|
||||
# Should print:
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
```
|
||||
|
||||
## Installing using Docker
|
||||
|
||||
This will use a Docker image that will isolate mediapipe's installation from the rest of the system.
|
||||
|
||||
1. [Install Docker](https://docs.docker.com/install/#supported-platforms) on
|
||||
your host system.
|
||||
|
||||
2. Build a docker image with tag "mediapipe".
|
||||
|
||||
```bash
|
||||
$ git clone https://github.com/google/mediapipe.git
|
||||
$ cd mediapipe
|
||||
$ docker build --tag=mediapipe .
|
||||
|
||||
# Should print:
|
||||
# Sending build context to Docker daemon 147.8MB
|
||||
# Step 1/9 : FROM ubuntu:latest
|
||||
# latest: Pulling from library/ubuntu
|
||||
# 6abc03819f3e: Pull complete
|
||||
# 05731e63f211: Pull complete
|
||||
# ........
|
||||
# See http://bazel.build/docs/getting-started.html to start a new project!
|
||||
# Removing intermediate container 82901b5e79fa
|
||||
# ---> f5d5f402071b
|
||||
# Step 9/9 : COPY . /mediapipe/
|
||||
# ---> a95c212089c5
|
||||
# Successfully built a95c212089c5
|
||||
# Successfully tagged mediapipe:latest
|
||||
```
|
||||
|
||||
3. Run the [Hello World desktop example](./hello_world_desktop.md).
|
||||
|
||||
```bash
|
||||
$ docker run -it --name mediapipe mediapipe:latest
|
||||
|
||||
root@bca08b91ff63:/mediapipe# GLOG_logtostderr=1 bazel run --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world
|
||||
|
||||
# Should print:
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
# Hello World!
|
||||
```
|
||||
|
||||
4. Build a MediaPipe Android example.
|
||||
|
||||
```bash
|
||||
$ docker run -it --name mediapipe mediapipe:latest
|
||||
|
||||
root@bca08b91ff63:/mediapipe# bash ./setup_android_sdk_and_ndk.sh
|
||||
|
||||
# Should print:
|
||||
# Android NDK is now installed. Consider setting $ANDROID_NDK_HOME environment variable to be /root/Android/Sdk/ndk-bundle/android-ndk-r18b
|
||||
# Set android_ndk_repository and android_sdk_repository in WORKSPACE
|
||||
# Done
|
||||
|
||||
root@bca08b91ff63:/mediapipe# bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu:objectdetectiongpu
|
||||
|
||||
# Should print:
|
||||
# Target //mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu:objectdetectiongpu up-to-date:
|
||||
# bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/objectdetectiongpu_deploy.jar
|
||||
# bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/objectdetectiongpu_unsigned.apk
|
||||
# bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/objectdetectiongpu.apk
|
||||
# INFO: Elapsed time: 144.462s, Critical Path: 79.47s
|
||||
# INFO: 1958 processes: 1 local, 1863 processwrapper-sandbox, 94 worker.
|
||||
# INFO: Build completed successfully, 2028 total actions
|
||||
```
|
||||
|
||||
<!-- 5. Uncomment the last line of the Dockerfile
|
||||
|
||||
```bash
|
||||
RUN bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/demo:object_detection_tensorflow_demo
|
||||
```
|
||||
|
||||
and rebuild the image and then run the docker image
|
||||
|
||||
```bash
|
||||
docker build --tag=mediapipe .
|
||||
docker run -i -t mediapipe:latest
|
||||
``` -->
|
||||
|
||||
[`WORKSPACE`]: https://github.com/google/mediapipe/blob/master/WORKSPACE
|
||||
[`opencv_linux.BUILD`]: https://github.com/google/mediapipe/tree/master/third_party/opencv_linux.BUILD
|
||||
[`opencv_macos.BUILD`]: https://github.com/google/mediapipe/tree/master/third_party/opencv_macos.BUILD
|
||||
[`ffmpeg_macos.BUILD`]:https://github.com/google/mediapipe/tree/master/third_party/ffmpeg_macos.BUILD
|
||||
[`setup_opencv.sh`]: https://github.com/google/mediapipe/blob/master/setup_opencv.sh
|
||||
@@ -1,12 +1,16 @@
|
||||
# Troubleshooting
|
||||
---
|
||||
layout: default
|
||||
title: Troubleshooting
|
||||
parent: Getting Started
|
||||
nav_order: 10
|
||||
---
|
||||
|
||||
- [Native method not found](#native-method-not-found)
|
||||
- [No registered calculator found](#no-registered-calculator-found)
|
||||
- [Out Of Memory error](#out-of-memory-error)
|
||||
- [Graph hangs](#graph-hangs)
|
||||
- [Calculator is scheduled infrequently](#calculator-is-scheduled-infrequently)
|
||||
- [Output timing is uneven](#output-timing-is-uneven)
|
||||
- [CalculatorGraph lags behind inputs](#calculatorgraph-lags-behind-inputs)
|
||||
# Troubleshooting
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
---
|
||||
|
||||
## Native method not found
|
||||
|
||||
@@ -41,7 +45,7 @@ When the library for a calculator is linked into an application binary, the
|
||||
calculator is automatically registered by name through the
|
||||
[`REGISTER_CALCULATOR`] macro using the [`registration.h`] library. Note that
|
||||
[`REGISTER_CALCULATOR`] can register a calculator with a namespace prefix,
|
||||
identical to its C++ namespace. In this case, the calcultor graph must also use
|
||||
identical to its C++ namespace. In this case, the calculator graph must also use
|
||||
the same namespace prefix.
|
||||
|
||||
## Out Of Memory error
|
||||
@@ -55,7 +59,7 @@ running MediaPipe graph. This can occur for a number of reasons, such as:
|
||||
|
||||
For problem (1), it may be necessary to drop some old packets in older to
|
||||
process the more recent packets. For some hints, see:
|
||||
[How to process realtime input streams](how_to_questions.md#how-to-process-realtime-input-streams)
|
||||
[`How to process realtime input streams`].
|
||||
|
||||
For problem (2), it could be that one input stream is lacking packets for some
|
||||
reason. A device or a calculator may be misconfigured or may produce packets
|
||||
@@ -63,7 +67,7 @@ only sporadically. This can cause downstream calculators to wait for many
|
||||
packets that will never arrive, which in turn causes packets to accumulate on
|
||||
some of their input streams. MediaPipe addresses this sort of problem using
|
||||
"timestamp bounds". For some hints see:
|
||||
[How to process realtime input streams](how_to_questions.md#how-to-process-realtime-input-streams)
|
||||
[`How to process realtime input streams`].
|
||||
|
||||
The MediaPipe setting [`CalculatorGraphConfig::max_queue_size`] limits the
|
||||
number of packets enqueued on any input stream by throttling inputs to the
|
||||
@@ -122,14 +126,14 @@ each packet as early as possible. Normally the lowest possible latency is the
|
||||
total time required by each calculator along a "critical path" of successive
|
||||
calculators. The latency of the a MediaPipe graph could be worse than the ideal
|
||||
due to delays introduced to display frames a even intervals as described in
|
||||
[Output timing is uneven](troubleshooting.md?cl=252235797#output-timing-is-uneven).
|
||||
[Output timing is uneven](#output-timing-is-uneven).
|
||||
|
||||
If some of the calculators in the graph cannot keep pace with the realtime input
|
||||
streams, then latency will continue to increase, and it becomes necessary to
|
||||
drop some input packets. The recommended technique is to use the MediaPipe
|
||||
calculators designed specifically for this purpose such as
|
||||
[`RealTimeFlowLimiterCalculator`] as described in
|
||||
[How to process realtime input streams](how_to_questions.md#how-to-process-realtime-input-streams).
|
||||
[`FlowLimiterCalculator`] as described in
|
||||
[`How to process realtime input streams`].
|
||||
|
||||
[`CalculatorGraphConfig`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator.proto
|
||||
[`CalculatorGraphConfig::max_queue_size`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator.proto
|
||||
@@ -141,4 +145,5 @@ calculators designed specifically for this purpose such as
|
||||
[`CalculatorGraph::WaitUntilDone`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_graph.h
|
||||
[`Timestamp::Done`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/timestamp.h
|
||||
[`CalculatorBase::Close`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.h
|
||||
[`RealTimeFlowLimiterCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/real_time_flow_limiter_calculator.cc
|
||||
[`FlowLimiterCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/flow_limiter_calculator.cc
|
||||
[`How to process realtime input streams`]: faq.md#how-to-process-realtime-input-streams
|
||||
|
After Width: | Height: | Size: 18 KiB |
|
After Width: | Height: | Size: 15 KiB |
|
After Width: | Height: | Size: 51 KiB |
|
After Width: | Height: | Size: 885 KiB |
|
After Width: | Height: | Size: 66 KiB |
|
After Width: | Height: | Size: 8.2 MiB |
|
After Width: | Height: | Size: 170 KiB |
|
After Width: | Height: | Size: 5.5 MiB |
|
After Width: | Height: | Size: 39 KiB |
|
After Width: | Height: | Size: 361 KiB |
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 15 KiB |
|
After Width: | Height: | Size: 5.8 KiB |
|
After Width: | Height: | Size: 11 KiB |
|
Before Width: | Height: | Size: 27 KiB After Width: | Height: | Size: 27 KiB |
|
After Width: | Height: | Size: 195 KiB |
|
Before Width: | Height: | Size: 35 KiB After Width: | Height: | Size: 35 KiB |
|
After Width: | Height: | Size: 118 KiB |
|
After Width: | Height: | Size: 7.1 MiB |
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.1 KiB |
|
After Width: | Height: | Size: 923 B |
|
Before Width: | Height: | Size: 46 KiB After Width: | Height: | Size: 46 KiB |
|
Before Width: | Height: | Size: 217 KiB After Width: | Height: | Size: 217 KiB |
|
After Width: | Height: | Size: 163 KiB |
|
After Width: | Height: | Size: 9.6 KiB |
|
After Width: | Height: | Size: 4.6 KiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
Before Width: | Height: | Size: 1.8 KiB After Width: | Height: | Size: 1.8 KiB |
|
After Width: | Height: | Size: 26 KiB |
|
After Width: | Height: | Size: 31 KiB |
|
After Width: | Height: | Size: 20 KiB |
|
After Width: | Height: | Size: 4.6 KiB |
|
After Width: | Height: | Size: 302 KiB |
|
After Width: | Height: | Size: 9.4 KiB |
|
After Width: | Height: | Size: 35 KiB |
|
After Width: | Height: | Size: 75 KiB |
|
After Width: | Height: | Size: 29 KiB |
|
After Width: | Height: | Size: 18 KiB |
|
After Width: | Height: | Size: 37 KiB |
|
After Width: | Height: | Size: 666 KiB |
|
After Width: | Height: | Size: 529 KiB |
|
After Width: | Height: | Size: 12 KiB |
|
After Width: | Height: | Size: 2.3 MiB |
|
After Width: | Height: | Size: 808 KiB |
|
After Width: | Height: | Size: 121 KiB |
|
After Width: | Height: | Size: 72 KiB |
|
After Width: | Height: | Size: 121 KiB |
|
After Width: | Height: | Size: 3.3 MiB |
|
After Width: | Height: | Size: 1.0 MiB |
|
After Width: | Height: | Size: 32 KiB |
|
After Width: | Height: | Size: 59 KiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 460 KiB |
|
After Width: | Height: | Size: 64 KiB |
|
After Width: | Height: | Size: 299 KiB |
|
After Width: | Height: | Size: 3.1 MiB |
|
After Width: | Height: | Size: 383 KiB |
|
After Width: | Height: | Size: 84 KiB |
|
After Width: | Height: | Size: 32 KiB |
|
After Width: | Height: | Size: 293 KiB |
|
After Width: | Height: | Size: 93 KiB |
|
After Width: | Height: | Size: 5.6 MiB |
|
After Width: | Height: | Size: 4.7 MiB |
|
After Width: | Height: | Size: 448 KiB |
|
After Width: | Height: | Size: 150 KiB |
|
After Width: | Height: | Size: 20 KiB |
|
After Width: | Height: | Size: 149 KiB |
|
After Width: | Height: | Size: 193 KiB |
|
After Width: | Height: | Size: 213 KiB |
|
After Width: | Height: | Size: 1.5 MiB |
|
After Width: | Height: | Size: 475 KiB |
|
After Width: | Height: | Size: 1.3 MiB |