Compare commits

..
Author SHA1 Message Date
Sebastian SchmidtandCopybara-Service 517e997179 Solve Linking error for Hello World iOS example
PiperOrigin-RevId: 515466634
2023-03-09 16:02:07 -08:00
MediaPipe TeamandCopybara-Service c2a69ab476 Update ImageFrameToGpuBufferCalculator to use api2 and GpuBuffer conversions
PiperOrigin-RevId: 515407159
2023-03-09 12:21:00 -08:00
MediaPipe TeamandCopybara-Service 5daf58009d internal
PiperOrigin-RevId: 515392932
2023-03-09 11:27:11 -08:00
Jiuqiang TangandCopybara-Service 5bd6a7082a Add requiredInputBufferSize as an input argument of createAudioRecord.
PiperOrigin-RevId: 515374407
2023-03-09 10:30:49 -08:00
MediaPipe TeamandCopybara-Service 563b193bca Bump Halide version from 14.0.0 to 15.0.0 and add MacOS Halide dependency
PiperOrigin-RevId: 515362310
2023-03-09 09:54:19 -08:00
MediaPipe TeamandCopybara-Service 2c64421471 Fix minor typos in MediaPipe synchronization description.
PiperOrigin-RevId: 515362264
2023-03-09 09:49:23 -08:00
MediaPipe TeamandCopybara-Service 39e2c8351f Add build system for Halide and expose FrameBufferUtils.
PiperOrigin-RevId: 515304264
2023-03-09 05:06:18 -08:00
Sebastian SchmidtandCopybara-Service 5398b8881d Internal change
PiperOrigin-RevId: 515225596
2023-03-08 21:04:55 -08:00
MediaPipe TeamandCopybara-Service c12eae229f Internal change
PiperOrigin-RevId: 515187906
2023-03-08 17:36:04 -08:00
MediaPipe TeamandCopybara-Service 8d9f627fd9 Internal change
PiperOrigin-RevId: 515187139
2023-03-08 17:32:21 -08:00
MediaPipe TeamandCopybara-Service 9f1f4273d0 Support hair segmentation model in image segmenter
PiperOrigin-RevId: 515151150
2023-03-08 15:01:14 -08:00
MediaPipe TeamandCopybara-Service 2fb62e4c29 Add volume_gain_db option to TensorsToAudioCalculator.
PiperOrigin-RevId: 515150234
2023-03-08 14:57:29 -08:00
Jiuqiang TangandCopybara-Service 253a5b477e Implement face stylizer graph and its C++ API.
PiperOrigin-RevId: 515139282
2023-03-08 14:18:08 -08:00
Kris TonthatandCopybara-Service 0a60c67667 Redirect "Getting started pages" from MediaPipe.dev to developers.google.com/mediapipe
PiperOrigin-RevId: 515137109
2023-03-08 14:10:45 -08:00
Yuqi LiandCopybara-Service 6f8c30abdd nit: update the description format in object_detector.MetadataWriter.
PiperOrigin-RevId: 515113799
2023-03-08 12:46:08 -08:00
MediaPipe TeamandCopybara-Service b8917ad31f Add FaceDetectorResult
PiperOrigin-RevId: 515104977
2023-03-08 12:10:13 -08:00
Sebastian SchmidtandCopybara-Service 09f63cbbe0 Internal change
PiperOrigin-RevId: 515094293
2023-03-08 11:33:38 -08:00
Jiuqiang TangandCopybara-Service ddc535e705 Add DftTensorFormat To TensorsToAudioCalculatorOptions.
PiperOrigin-RevId: 515077766
2023-03-08 10:38:40 -08:00
Hadon NashandCopybara-Service 16c2e32a0d Internal change
PiperOrigin-RevId: 515044500
2023-03-08 08:29:53 -08:00
MediaPipe TeamandCopybara-Service 13d7e1a7ba Changes the visibility of ImageTensorSpecs to public to allow clients use GetInputImageTensorSpecs().
PiperOrigin-RevId: 514853148
2023-03-07 15:16:40 -08:00
MediaPipe TeamandCopybara-Service 225dd36a63 Internal change
PiperOrigin-RevId: 514835354
2023-03-07 14:11:47 -08:00
Yuqi LiandCopybara-Service 46b5c4012d add custom metadata in metadata_schema.
PiperOrigin-RevId: 514768756
2023-03-07 10:15:09 -08:00
MediaPipe TeamandCopybara-Service 414031867e Internal change
PiperOrigin-RevId: 514763674
2023-03-07 10:00:10 -08:00
Jiuqiang TangandCopybara-Service dbd6d72696 Add GetInputImageTensorSpecs into BaseVisionTaskApi for tasks api users to get input image tensor specifications.
PiperOrigin-RevId: 514650593
2023-03-07 00:44:07 -08:00
MediaPipe TeamandCopybara-Service 2f2a74da6a Internal MediaPipe Tasks change.
PiperOrigin-RevId: 514637484
2023-03-06 23:17:35 -08:00
MediaPipe TeamandCopybara-Service c1b460920c normalized keypoint container. Detection adds optional keypoint list.
PiperOrigin-RevId: 514624925
2023-03-06 22:02:59 -08:00
MediaPipe TeamandCopybara-Service e4ac1a10c2 TensorsToSegmentation web postprocessing on GPU: softmax activation
PiperOrigin-RevId: 514589911
2023-03-06 19:08:53 -08:00
MediaPipe TeamandCopybara-Service bd9a2ee1fc Internal change
PiperOrigin-RevId: 514580892
2023-03-06 18:13:50 -08:00
MediaPipe TeamandCopybara-Service 0337c7f52f Fix typo in ImageSegmenter.java
PiperOrigin-RevId: 514534013
2023-03-06 14:51:49 -08:00
MediaPipe TeamandCopybara-Service f15244997f Internal change
PiperOrigin-RevId: 514498364
2023-03-06 12:58:01 -08:00
Jiuqiang TangandCopybara-Service 77fcaa9597 Allow TensorsToAudioCalculator to add and overlay adjacent windows.
PiperOrigin-RevId: 514483756
2023-03-06 12:08:57 -08:00
MediaPipe TeamandCopybara-Service c64f83fff4 Internal change
PiperOrigin-RevId: 514452059
2023-03-06 10:29:29 -08:00
MediaPipe TeamandCopybara-Service 6c68d4c358 Internal change
PiperOrigin-RevId: 514421618
2023-03-06 08:42:19 -08:00
MediaPipe TeamandCopybara-Service 96b2958f18 Add ViewProvider for ImageFrame in GpuBufferStorageYuvImage.
PiperOrigin-RevId: 514344066
2023-03-06 07:53:02 -08:00
MediaPipe TeamandCopybara-Service a43be73ee4 Internal MediaPipe Tasks change.
PiperOrigin-RevId: 514150403
2023-03-06 07:32:59 -08:00
MediaPipe TeamandCopybara-Service 763842289a Add FaceLandmarkerResult for FaceLandmarker API
PiperOrigin-RevId: 514137566
2023-03-06 07:31:30 -08:00
MediaPipe TeamandCopybara-Service cd14d2e688 Internal MediaPipe Tasks change.
PiperOrigin-RevId: 514002859
2023-03-06 07:29:55 -08:00
Yuqi LiandCopybara-Service dbe4175a08 Internal change
PiperOrigin-RevId: 514001732
2023-03-06 07:28:20 -08:00
MediaPipe TeamandCopybara-Service c98b4b6ec6 Internal change
PiperOrigin-RevId: 513992315
2023-03-06 07:26:39 -08:00
MediaPipe TeamandCopybara-Service 3d41eabc2e Face Detector C++ API
PiperOrigin-RevId: 513959349
2023-03-06 07:17:46 -08:00
MediaPipe TeamandCopybara-Service 5b2678a49f Add a CHECK for the cases when null service is accessed unconditionally.
PiperOrigin-RevId: 513956583
2023-03-06 07:16:29 -08:00
MediaPipe TeamandCopybara-Service 9c3abcd06f Document graph service usage with docs and unit tests.
PiperOrigin-RevId: 513955877
2023-03-06 07:15:10 -08:00
MediaPipe TeamandCopybara-Service 3854d9fcd5 Document proper way to set services.
PiperOrigin-RevId: 513955214
2023-03-06 07:13:54 -08:00
MediaPipe TeamandCopybara-Service 2963739086 Internal MediaPipe Tasks change.
PiperOrigin-RevId: 513897822
2023-03-06 07:12:37 -08:00
Yuqi LiandCopybara-Service fe92d2e781 Metadata Writer: add object detection metadata writer.
PiperOrigin-RevId: 513897494
2023-03-06 07:11:10 -08:00
Sebastian SchmidtandCopybara-Service 13db1c55d3 Make Bert/RegexTokenizerTest pass on Windows
PiperOrigin-RevId: 513887535
2023-03-06 07:04:30 -08:00
MediaPipe TeamandCopybara-Service b7ec83efb5 Add optional face blendshapes to face landmarks detector graph.
PiperOrigin-RevId: 513867488
2023-03-03 10:48:55 -08:00
Hadon NashandCopybara-Service c9c1bf21ae Internal change
PiperOrigin-RevId: 513854481
2023-03-03 10:00:10 -08:00
MediaPipe TeamandCopybara-Service 3837c92fd5 Add functions for checking the existence of options in subgraphs and calculators.
PiperOrigin-RevId: 513689742
2023-03-02 18:01:07 -08:00
MediaPipe TeamandCopybara-Service 91d53cd181 Make image preprocessing support RGBA image.
PiperOrigin-RevId: 513680594
2023-03-02 17:13:19 -08:00
Sebastian SchmidtandCopybara-Service c3adf19706 Invoke vector listeners for zero-element vectors
PiperOrigin-RevId: 513662913
2023-03-02 15:59:26 -08:00
Sebastian SchmidtandCopybara-Service 7664e0ef64 Make SentencePieceTokenizerTest work on Windows
PiperOrigin-RevId: 513638600
2023-03-02 14:23:22 -08:00
MediaPipe TeamandCopybara-Service 945b36766c Let web API add empty packets to input streams
PiperOrigin-RevId: 513630040
2023-03-02 13:51:39 -08:00
MediaPipe TeamandCopybara-Service 0ba35cf1a7 Internal change
PiperOrigin-RevId: 513608516
2023-03-02 12:29:06 -08:00
MediaPipe TeamandCopybara-Service c71f78c38f Internal change
PiperOrigin-RevId: 513607664
2023-03-02 12:24:51 -08:00
MediaPipe TeamandCopybara-Service dd4301802a migrate env_generator_calculator
PiperOrigin-RevId: 513585830
2023-03-02 11:05:02 -08:00
MediaPipe TeamandCopybara-Service 02ee934b3e When "--define=MEDIAPIPE_NO_JNI=1" used in compilation, no implementation in libandroid.so is used.
PiperOrigin-RevId: 513580575
2023-03-02 10:46:35 -08:00
MediaPipe TeamandCopybara-Service 6796977a04 migrate geometry_pipeline_calculator
PiperOrigin-RevId: 513368520
2023-03-01 16:09:48 -08:00
Sebastian SchmidtandCopybara-Service f15637b62c Make Metadata tests pass on Windows
PiperOrigin-RevId: 513365428
2023-03-01 15:59:43 -08:00
Copybara-Service 82ee00f25d Merge pull request #4118 from kuaashish:master
PiperOrigin-RevId: 513364683
2023-03-01 15:55:37 -08:00
Sebastian SchmidtandCopybara-Service abfcd8ec1d Make LoadBinaryContent work on Windows
PiperOrigin-RevId: 513330348
2023-03-01 13:49:02 -08:00
Joe FernandezandCopybara-Service 07fa5c2fc8 Internal change
PiperOrigin-RevId: 513300452
2023-03-01 11:56:26 -08:00
MediaPipe TeamandCopybara-Service 22fce9e136 migrate mediapipe/modules/face_geometry to mediapipe/tasks
PiperOrigin-RevId: 513284254
2023-03-01 10:59:56 -08:00
Joe FernandezandCopybara-Service 0a937eba98 Internal change
PiperOrigin-RevId: 513255798
2023-03-01 09:21:25 -08:00
Sebastian SchmidtandCopybara-Service 22f186724e Make delegate sticky
PiperOrigin-RevId: 513249729
2023-03-01 09:00:36 -08:00
Sebastian SchmidtandCopybara-Service 4a1ba11e3f Do not use designated initializers
PiperOrigin-RevId: 513028900
2023-02-28 13:52:36 -08:00
Jiuqiang TangandCopybara-Service 172b360478 Remove unused variable 'input_num'.
PiperOrigin-RevId: 513009671
2023-02-28 12:37:03 -08:00
MediaPipe TeamandCopybara-Service 2143baf7d5 Dos/Donts: separating nodes for readability
PiperOrigin-RevId: 512994288
2023-02-28 11:40:03 -08:00
Copybara-Service f1b20b0c52 Merge pull request #4104 from priankakariatyml:ios-image-classifier
PiperOrigin-RevId: 512991238
2023-02-28 11:29:19 -08:00
Jiuqiang TangandCopybara-Service 646ab8505a Small fixes to TensorsToImageCalculator.
PiperOrigin-RevId: 512967357
2023-02-28 10:11:42 -08:00
Sebastian SchmidtandCopybara-Service 06dd003113 Internal change
PiperOrigin-RevId: 512777889
2023-02-27 17:34:29 -08:00
MediaPipe TeamandCopybara-Service bfddb32358 remove usage of var for ImageSegmenter.java
PiperOrigin-RevId: 512767947
2023-02-27 16:49:27 -08:00
Kris TonthatandCopybara-Service 2cbaf84f50 Internal change
PiperOrigin-RevId: 512758617
2023-02-27 16:12:26 -08:00
Sebastian SchmidtandCopybara-Service 689d5ba6e3 Internal
PiperOrigin-RevId: 512755713
2023-02-27 16:00:35 -08:00
Sebastian SchmidtandCopybara-Service 1dca871d09 Remove designated initializer use from CombinedPredictionCalculatorTestCase
PiperOrigin-RevId: 512747781
2023-02-27 15:28:34 -08:00
Sebastian SchmidtandCopybara-Service 1d927c5971 Internal change
PiperOrigin-RevId: 512709627
2023-02-27 13:08:04 -08:00
MediaPipe TeamandCopybara-Service a60d67eb10 Update ImageSegmenter API for image/video mode to have both callback API and returned result API.
PiperOrigin-RevId: 512697585
2023-02-27 12:21:30 -08:00
Jiuqiang TangandCopybara-Service aa61abe386 Implement TensorsToImageCalculator.
PiperOrigin-RevId: 512687103
2023-02-27 11:46:07 -08:00
MediaPipe TeamandCopybara-Service 5913e5dd1d Internal change
PiperOrigin-RevId: 512686714
2023-02-27 11:41:44 -08:00
MediaPipe TeamandCopybara-Service 75576700ed Add Bitmap image capture capability to GlSurfaceViewRenderer.
PiperOrigin-RevId: 512677893
2023-02-27 11:14:20 -08:00
Sebastian SchmidtandCopybara-Service 9f59d4d01b Remove cosineSimilarity() from AudioEmbedder
PiperOrigin-RevId: 512671255
2023-02-27 11:13:20 -08:00
MediaPipe TeamandCopybara-Service 39b2fec60f Internal change
PiperOrigin-RevId: 512643827
2023-02-27 11:12:20 -08:00
MediaPipe TeamandCopybara-Service 1575e7f79c Internal change
PiperOrigin-RevId: 512457466
2023-02-27 11:11:24 -08:00
MediaPipe TeamandCopybara-Service 17466fb7f1 Internal change
PiperOrigin-RevId: 512111461
2023-02-27 11:10:19 -08:00
MediaPipe Teamandjqtang 9054ff7283 Internal change
PiperOrigin-RevId: 511928038
2023-02-27 11:02:58 -08:00
MediaPipe TeamandCopybara-Service 01c64082f1 ImageToTensorCalculator decides the output tensor size in runtime from the input image size.
PiperOrigin-RevId: 511882195
2023-02-23 14:18:31 -08:00
MediaPipe TeamandCopybara-Service 0981367e84 DetectionTransformationCalculator handles empty input.
PiperOrigin-RevId: 511881969
2023-02-23 14:14:20 -08:00
Kris TonthatandCopybara-Service 0b989f8c8d Add legacy solutions note
PiperOrigin-RevId: 511880995
2023-02-23 14:09:41 -08:00
kuaashishandGitHub 86a2fa300b Update iris.md 2023-02-23 22:08:05 +05:30
kuaashishandGitHub f42b60e08f Update iris.md 2023-02-23 22:07:40 +05:30
kuaashishandGitHub 1c37010dde Update objectron.md 2023-02-23 22:07:04 +05:30
kuaashishandGitHub 48d72e7ddc Update face_mesh.md 2023-02-23 22:06:35 +05:30
kuaashishandGitHub 35d1b98433 Update objectron.md 2023-02-23 22:04:49 +05:30
kuaashishandGitHub 34e361f81e Update media_sequence.md 2023-02-23 22:04:02 +05:30
kuaashishandGitHub 488b1e23a0 Update troubleshooting.md 2023-02-23 22:01:58 +05:30
kuaashishandGitHub 9e37b520fb Update faq.md 2023-02-23 22:01:19 +05:30
kuaashishandGitHub abc8c73aee Update javascript.md 2023-02-23 22:00:01 +05:30
kuaashishandGitHub bc157ff990 Update help.md 2023-02-23 21:57:42 +05:30
MediaPipe TeamandCopybara-Service 75d799d2b9 Add Copyright head
PiperOrigin-RevId: 511694458
2023-02-22 22:47:02 -08:00
MediaPipe TeamandCopybara-Service ff10f3c814 Move NormalizedKeypoint to a separate header
PiperOrigin-RevId: 511642586
2023-02-22 17:42:06 -08:00
MediaPipe TeamandCopybara-Service 2c3c1e664a add TensorsToFaceLandmarksGraph to support two types of face mesh models.
PiperOrigin-RevId: 511626032
2023-02-22 16:22:13 -08:00
Yuqi LiandCopybara-Service 9e7950a69a Make each model file in the model asset bundle to be aligned relative to the start of the file (alignment = 4).
PiperOrigin-RevId: 511624410
2023-02-22 16:15:24 -08:00
MediaPipe TeamandCopybara-Service 40b0dc960a Internal Changes
PiperOrigin-RevId: 511604893
2023-02-22 14:55:03 -08:00
MediaPipe TeamandCopybara-Service 000aeeb036 Update face detector graph for downstream face landmarks graph.
PiperOrigin-RevId: 511566984
2023-02-22 12:31:57 -08:00
MediaPipe TeamandCopybara-Service fbbc13d756 update detection result to include optional keypoints.
PiperOrigin-RevId: 511538880
2023-02-22 10:50:58 -08:00
MediaPipe TeamandCopybara-Service c026f935bc Add @ to all references to files in WORKSPACE.bazel.
PiperOrigin-RevId: 511504265
2023-02-22 08:36:36 -08:00
MediaPipe TeamandCopybara-Service e2ac704428 internal change
PiperOrigin-RevId: 511338483
2023-02-21 16:40:37 -08:00
MediaPipe TeamandCopybara-Service 9a6d5e0444 Update MediaPipe TFLite code to use generic "shim" symbols and headers
PiperOrigin-RevId: 511265313
2023-02-21 11:58:32 -08:00
Prianka Liz Kariat 68fdf6b6cb Updated comment arguments in MPPImageUtils 2023-02-20 11:27:21 +05:30
Prianka Liz Kariat 4d2dd50703 Updated method name in MPPVisionPacketCreator 2023-02-20 11:23:04 +05:30
Jiuqiang TangandCopybara-Service 8a0830ce7b Add "java_package" and "java_outer_classname" to ImageTransformationCalculatorOptions.
PiperOrigin-RevId: 510720513
2023-02-18 16:58:56 -08:00
MediaPipe TeamandCopybara-Service d0aa98bd5f Internal change
PiperOrigin-RevId: 510570555
2023-02-17 18:50:05 -08:00
MediaPipe TeamandCopybara-Service 68ba9a6ebf [ImageTransformationCalculator]Add option to specify padding color
PiperOrigin-RevId: 510541473
2023-02-17 15:57:19 -08:00
MediaPipe TeamandCopybara-Service 37a825c98d graph utils functions.
PiperOrigin-RevId: 510513793
2023-02-17 13:54:46 -08:00
MediaPipe TeamandCopybara-Service 1a60a0e2d6 Use box_output_format to decide keypoint order
PiperOrigin-RevId: 510464960
2023-02-17 10:30:57 -08:00
MediaPipe TeamandCopybara-Service 223c504d81 Add note to installation doc for Debian distros
PiperOrigin-RevId: 510317890
2023-02-16 20:27:59 -08:00
MediaPipe TeamandCopybara-Service 4ed9f507ad Internal change
PiperOrigin-RevId: 510281435
2023-02-16 17:01:56 -08:00
Hadon NashandCopybara-Service 5f6bbdbce5 Internal change
PiperOrigin-RevId: 510243967
2023-02-16 14:26:46 -08:00
MediaPipe TeamandCopybara-Service 1b66fc2802 Fix Typo.
PiperOrigin-RevId: 510156746
2023-02-16 08:55:56 -08:00
MediaPipe TeamandCopybara-Service 152f66dfd3 This will fix the command to clone mediapipe in the Debian and Ubuntu, CentOS, Windows, Windows Subsystem for Linux (WSL), Docker. Doc command mentioned
is throwing error when directly copied and pasted in the terminal.

PiperOrigin-RevId: 510061887
2023-02-16 00:23:49 -08:00
MediaPipe TeamandCopybara-Service ba10ae8410 Add mediapipe tasks face blendshapes graph
PiperOrigin-RevId: 509995915
2023-02-15 19:26:56 -08:00
MediaPipe TeamandCopybara-Service bdd1c24990 Refactor common methods into vision/core/image_utils.py and vision/core/test_utils.py
PiperOrigin-RevId: 509968910
2023-02-15 16:54:38 -08:00
MediaPipe TeamandCopybara-Service 3d4ed305bc Make Tensor::Move() forward quantization parameters.
PiperOrigin-RevId: 509951145
2023-02-15 15:40:22 -08:00
MediaPipe TeamandCopybara-Service 796a96d842 Add support for [xmin, ymin, xmax, ymax] style of bbox output
PiperOrigin-RevId: 509942540
2023-02-15 15:06:52 -08:00
Prianka Liz Kariat 5e5a1a733f Updated imports 2023-02-16 01:37:50 +05:30
Prianka Liz Kariat dce81342b6 Updated imports of MPPImageUtils 2023-02-16 01:35:27 +05:30
Prianka Liz Kariat ae05c78443 Updated formatting 2023-02-16 01:33:33 +05:30
Prianka Liz Kariat 42e35503d9 Removed unwanted declarations in namespace 2023-02-16 01:31:51 +05:30
Prianka Liz Kariat a503fb53e0 Updated MPPVisionTaskRunner 2023-02-16 01:31:29 +05:30
Prianka Liz Kariat a0253274cc Added MPPVisionPacketCreator 2023-02-16 01:31:09 +05:30
Prianka Liz Kariat 8c3e3456a3 Added MPPImageClassifierOptions 2023-02-16 01:26:10 +05:30
Prianka Liz Kariat 825b30bccd Added MPPImageClassifierResult 2023-02-16 01:25:57 +05:30
Prianka Liz Kariat a128810564 Updated MPPImageUtils with methods to create image frame 2023-02-16 01:25:16 +05:30
MediaPipe TeamandCopybara-Service 40c3e72c9c Added note to [page](https://google.github.io/mediapipe/getting_started/ios.html#create-an-xcode-project) that the xcode version in script should be same as installed version
PiperOrigin-RevId: 509801311
2023-02-15 05:09:31 -08:00
Sebastian SchmidtandCopybara-Service d450a673c0 Open visibility of iOS TextClassifier & TextEmbedder
PiperOrigin-RevId: 509645638
2023-02-14 14:53:24 -08:00
MediaPipe TeamandCopybara-Service 63a57e2da3 Graph builder Dos and Donts / Nodes minimal direct usage
PiperOrigin-RevId: 509632450
2023-02-14 14:16:18 -08:00
MediaPipe TeamandCopybara-Service 9b1fe849d5 Fix incorrect comments in docs code sections
PiperOrigin-RevId: 509632192
2023-02-14 14:11:48 -08:00
MediaPipe TeamandCopybara-Service d9ac3876de Dos and Donts / Graph outputs + small adjustments for previous section (e.g removing "simply" word)
PiperOrigin-RevId: 509632077
2023-02-14 14:06:48 -08:00
MediaPipe TeamandCopybara-Service 5f2261ff59 face landmarks detector graph
PiperOrigin-RevId: 509630430
2023-02-14 14:00:37 -08:00
Copybara-Service d6fd2c52a7 Merge pull request #4093 from priankakariatyml:ios-ml-image-utils
PiperOrigin-RevId: 509602397
2023-02-14 12:17:41 -08:00
Sebastian SchmidtandCopybara-Service c2260e8877 Use mediapipe_proto_library for gl_context_options_proto
PiperOrigin-RevId: 509601125
2023-02-14 12:13:01 -08:00
MediaPipe TeamandCopybara-Service c66b7e83f0 ssd anchors calculator add fixed anchors.
PiperOrigin-RevId: 509574814
2023-02-14 10:43:23 -08:00
Prianka Liz Kariat 55e5377c08 Updated formatting 2023-02-14 20:51:51 +05:30
Prianka Liz Kariat 3a7aa29226 Updated note about packets callback in vision task runner 2023-02-14 20:49:14 +05:30
Prianka Liz Kariat 4e1fa82d50 Added designated initializer in vision task runner 2023-02-14 20:40:21 +05:30
Prianka Liz Kariat a490255c17 Updated formatting 2023-02-14 20:40:06 +05:30
Prianka Liz Kariat 17ff493a9d Updated formatting 2023-02-14 20:39:25 +05:30
Prianka Liz Kariat b940a19462 Merge branch 'master' into ios-ml-image-utils 2023-02-14 20:34:55 +05:30
Copybara-Service a578a702ec Merge pull request #4075 from priankakariatyml:ios-ml-image
PiperOrigin-RevId: 509465929
2023-02-14 02:19:35 -08:00
MediaPipe TeamandCopybara-Service ef9e5f881b Add Dos and Donts Section / Graph inputs
PiperOrigin-RevId: 509347658
2023-02-13 15:26:23 -08:00
MediaPipe TeamandCopybara-Service caa7d85141 Improves page-contents navigation: adds Uility Functions/Classes to page-contents)
PiperOrigin-RevId: 509347516
2023-02-13 15:22:13 -08:00
MediaPipe TeamandCopybara-Service 513ab7abba Internal change
PiperOrigin-RevId: 509338497
2023-02-13 14:47:34 -08:00
Sebastian SchmidtandCopybara-Service deae714a5c Enable Python Audio Classifier & Embedder on Windows
PiperOrigin-RevId: 509243198
2023-02-13 09:06:19 -08:00
Prianka Liz Kariat 1cd3c05fc5 Added default case to running mode check 2023-02-13 19:58:14 +05:30
Prianka Liz Kariat 328fe4ed39 Added vision task runner 2023-02-13 19:55:12 +05:30
Prianka Liz Kariat 1623b591b7 Added MPPRunningMode 2023-02-13 19:52:00 +05:30
Prianka Liz Kariat e2f1d07c8b Updated target definition for MPPImage 2023-02-13 19:44:09 +05:30
Prianka Liz Kariat 02734b3add Added MPPImageUtils 2023-02-13 19:43:49 +05:30
Prianka Liz Kariat acba3ab26d Bug fixes in MPPImage 2023-02-13 19:41:15 +05:30
Copybara-Service 5afa815bef Merge pull request #4050 from kuaashish:patch-3
PiperOrigin-RevId: 509190256
2023-02-13 04:34:51 -08:00
MediaPipe TeamandCopybara-Service 626f92caea Move BuildInputImageTensorSpecs to utils
PiperOrigin-RevId: 508829724
2023-02-10 21:57:13 -08:00
MediaPipe TeamandCopybara-Service 2c82f67097 Add location info in registry (debug mode only)
PiperOrigin-RevId: 508786558
2023-02-10 16:58:44 -08:00
MediaPipe TeamandCopybara-Service 23012f2151 Add Advanced Usage/Utility Classes section in Building Graph in C++
PiperOrigin-RevId: 508776246
2023-02-10 16:06:09 -08:00
MediaPipe TeamandCopybara-Service 482ee8f96c Add Capabilities/Utility Functions section in Building Graph in C++
PiperOrigin-RevId: 508773788
2023-02-10 15:54:23 -08:00
Hadon NashandCopybara-Service e719d2d423 Internal change
PiperOrigin-RevId: 508766449
2023-02-10 15:23:01 -08:00
Aden GrueandCopybara-Service 09ad961421 Internal change
PiperOrigin-RevId: 508701833
2023-02-10 10:54:55 -08:00
MediaPipe TeamandCopybara-Service 39a943e91e Request ModelResourcesGraphService for all Task API web graphs.
PiperOrigin-RevId: 508681004
2023-02-10 09:36:03 -08:00
MediaPipe TeamandCopybara-Service 915d2c7417 Internal text task change.
PiperOrigin-RevId: 508568811
2023-02-09 22:31:14 -08:00
MediaPipe TeamandCopybara-Service d61b7dbef8 Updated the Begin/EndLoopCalculator to be able to handle mediapipe::Tensor
type.

PiperOrigin-RevId: 508552066
2023-02-09 20:33:40 -08:00
MediaPipe TeamandCopybara-Service 8a49a5f822 This will fix the command to clone mediapipe in the macOS. Doc command mentioned
is throwing error when directly copied and pasted in the terminal.

PiperOrigin-RevId: 508550373
2023-02-09 20:23:58 -08:00
MediaPipe TeamandCopybara-Service 1eb54bdaa6 Internal change
PiperOrigin-RevId: 508533812
2023-02-09 18:49:01 -08:00
MediaPipe TeamandCopybara-Service 1f848c4d69 Fix typo in CalculatorGraph error message
PiperOrigin-RevId: 508517972
2023-02-09 17:29:33 -08:00
MediaPipe TeamandCopybara-Service be0681c61d Adds "Building Graphs in C++" initial page and updates "Graph" page to link to that section showcasing alternative C++ graph representation.
PiperOrigin-RevId: 508517348
2023-02-09 17:22:44 -08:00
MediaPipe TeamandCopybara-Service 2163920ee8 Minor updates to MediaPipe graph documentation: use Graph/CalculatorGraphConfig instead of GraphConfig and mention that node can be a subgraph.
PiperOrigin-RevId: 508517281
2023-02-09 17:21:30 -08:00
MediaPipe TeamandCopybara-Service fd764dae0a Depends on TFLite shim header.
PiperOrigin-RevId: 508491302
2023-02-09 15:29:47 -08:00
MediaPipe TeamandCopybara-Service 99fc975f49 Add download model on demand to text classifier
PiperOrigin-RevId: 508441452
2023-02-09 12:15:17 -08:00
MediaPipe TeamandCopybara-Service 28f728bed5 Change gesture_recognizer to download model files on-demand from GCS.
PiperOrigin-RevId: 508424508
2023-02-09 11:13:06 -08:00
MediaPipe TeamandCopybara-Service c3907229fe Use case: add metadata to headerless graph input audio streams.
PiperOrigin-RevId: 508389799
2023-02-09 09:07:02 -08:00
MediaPipe TeamandCopybara-Service 598d44a27d Internal change
PiperOrigin-RevId: 508284935
2023-02-08 23:31:41 -08:00
MediaPipe TeamandCopybara-Service 6ea2d579e1 Internal change
PiperOrigin-RevId: 508173086
2023-02-08 13:51:14 -08:00
Sebastian SchmidtandCopybara-Service 6c4ebd2d93 Pin version for com_google_audio_tools
PiperOrigin-RevId: 508158159
2023-02-08 12:53:39 -08:00
Rachel HornungandCopybara-Service 4b28ced8fd #MediaSequence Prevent undefined index access in metadata reconciliation.
Reconciliation required feature_lists.feature_list[0].value.feature[0].float_list.size_>=1, now we skip keys where size_=0.

PiperOrigin-RevId: 508095655
2023-02-08 09:06:02 -08:00
Prianka Liz Kariat 0e944cb764 Merged ios-ml-image with master 2023-02-08 21:43:39 +05:30
Prianka Liz Kariat e1ca81c352 Added objective c tests for MPPImage 2023-02-08 21:40:41 +05:30
Prianka Liz Kariat a82ffcd97c Updated mppimage target 2023-02-08 21:40:23 +05:30
Prianka Liz Kariat 445876d96b Updated formatting 2023-02-08 21:35:15 +05:30
Copybara-Service 19d895eb73 Merge pull request #4057 from kuaashish:master
PiperOrigin-RevId: 507869364
2023-02-07 13:29:56 -08:00
Copybara-Service fd6dfcfcab Merge pull request #4066 from priankakariatyml:ios-ml-image
PiperOrigin-RevId: 507844473
2023-02-07 11:52:37 -08:00
Sebastian SchmidtandCopybara-Service 712a22101f Do not use designated initializer
PiperOrigin-RevId: 507805920
2023-02-07 09:37:46 -08:00
Sebastian SchmidtandCopybara-Service e8caaeed61 Update WASM files for 0.1.0-alpha-4 release
PiperOrigin-RevId: 507792684
2023-02-07 08:46:15 -08:00
Prianka Liz Kariat be829eb276 Updated name of MPPTaskImage 2023-02-07 19:17:00 +05:30
Jiuqiang TangandCopybara-Service 01c6a8b49b Add volume_gain_db option into AudioToTensorCalculator.
PiperOrigin-RevId: 507748012
2023-02-07 05:14:42 -08:00
MediaPipe TeamandCopybara-Service 9b040630a3 Updating the Javascript API's FaceDetectionOptions since modelSelection is not a valid option for setOptions().
PiperOrigin-RevId: 507664805
2023-02-06 20:52:12 -08:00
Copybara-Service 28c07430ba Merge pull request #4052 from priankakariatyml:ios-text-embedder
PiperOrigin-RevId: 507602101
2023-02-06 15:42:48 -08:00
Chris McClanahanandCopybara-Service e2ef78433f Add more filtering methods to detection filter calculator.
PiperOrigin-RevId: 507581281
2023-02-06 14:19:40 -08:00
Jiuqiang TangandCopybara-Service f4b0cf1cff Internal change
PiperOrigin-RevId: 507540898
2023-02-06 11:48:51 -08:00
Sebastian SchmidtandCopybara-Service daf0a76c87 Update TensorFlow to latest
PiperOrigin-RevId: 507505016
2023-02-06 09:44:19 -08:00
MediaPipe TeamandCopybara-Service 5a77339780 Internal change
PiperOrigin-RevId: 507495569
2023-02-06 09:05:12 -08:00
Prianka Liz Kariat a3889c946d Updated list of designated initializers 2023-02-06 16:40:36 +05:30
Prianka Liz Kariat 0e4f0fbf9b Updated file name for MPPTaskImage 2023-02-06 16:39:17 +05:30
Prianka Liz Kariat 4da91896af Updated swift name for ImageSource Type 2023-02-06 16:36:33 +05:30
Prianka Liz Kariat a5ac5b9eec Updated type name of image source 2023-02-06 16:35:27 +05:30
Prianka Liz Kariat cbccd472ab Added MPPTaskImage for iOS vision tasks 2023-02-06 16:29:53 +05:30
MediaPipe TeamandCopybara-Service 4d8af4315f Rollback: Add PacketSharingOwnership, a safer replacement for PointToForeign.
PiperOrigin-RevId: 507288476
2023-02-05 08:59:09 -08:00
Jiuqiang TangandCopybara-Service 8e097ea9c3 Internal changes
PiperOrigin-RevId: 507016761
2023-02-03 16:07:42 -08:00
Sebastian SchmidtandCopybara-Service 680cfcc99b Update MP Tasks to observe timestamp bounds
PiperOrigin-RevId: 507006313
2023-02-03 15:21:41 -08:00
MediaPipe TeamandCopybara-Service 01834a5cdf Internal change
PiperOrigin-RevId: 506982474
2023-02-03 13:39:02 -08:00
Sebastian SchmidtandCopybara-Service 046efddf8b Add Empty Packet support to GraphRunner
PiperOrigin-RevId: 506963970
2023-02-03 12:17:00 -08:00
MediaPipe TeamandCopybara-Service 632a3602dd Internal text tasks change.
PiperOrigin-RevId: 506957718
2023-02-03 11:52:09 -08:00
MediaPipe TeamandCopybara-Service 386445c8dd Add ViewProvider<FrameBuffer> to YuvImage storage backend.
PiperOrigin-RevId: 506881556
2023-02-03 05:58:29 -08:00
kuaashishandGitHub e7dff428fa Delete 40-feature-request.md 2023-02-03 19:16:46 +05:30
kuaashishandGitHub 9b7081837c Delete 30-bug-issue.md 2023-02-03 19:16:35 +05:30
kuaashishandGitHub dd8d8ae422 Delete 20-documentation-issue.md 2023-02-03 19:16:26 +05:30
kuaashishandGitHub 4e47081d12 Delete 14-studio-issue.md 2023-02-03 19:16:16 +05:30
kuaashishandGitHub d3bd5db1f8 Delete 13-solution-issue.md 2023-02-03 19:16:01 +05:30
kuaashishandGitHub 1b535a4fd1 Delete 12-model-maker-issue.md 2023-02-03 19:15:52 +05:30
kuaashishandGitHub 51490ea5d9 Delete 11-tasks-issue.md 2023-02-03 19:15:44 +05:30
kuaashishandGitHub 2e8d615153 Rename model_maker_issue_template1.yaml to model_maker_issue_template.yaml 2023-02-03 19:07:59 +05:30
kuaashishandGitHub 77f3cd8942 Rename build.yaml to build.install_issue_template.yaml 2023-02-03 19:06:09 +05:30
kuaashishandGitHub b10812a7e2 Update Documentation_issue_template.yaml 2023-02-03 18:27:16 +05:30
kuaashishandGitHub 5046c2c5ce Update Documentation_issue_template.yaml 2023-02-03 18:26:53 +05:30
kuaashishandGitHub c0a360d8f4 Update Documentation_issue_template.yaml 2023-02-03 18:26:41 +05:30
kuaashishandGitHub 681136a1d8 Update Documentation_issue_template.yaml 2023-02-03 18:25:47 +05:30
kuaashishandGitHub 212b0279d0 Update Documentation_issue_template.yaml 2023-02-03 18:24:36 +05:30
kuaashishandGitHub 57b84d8ab3 Update Documentation_issue_template.yaml 2023-02-03 18:12:23 +05:30
kuaashishandGitHub 00592ac22d Update Documentation_issue_template.yaml 2023-02-03 18:11:55 +05:30
kuaashishandGitHub 423359b478 Update Documentation_issue_template.yaml 2023-02-03 18:10:47 +05:30
kuaashishandGitHub ba4b5160cd Update Documentation_issue_template.yaml 2023-02-03 18:10:18 +05:30
kuaashishandGitHub a2189c0143 Update Documentation_issue_template.yaml 2023-02-03 18:09:16 +05:30
kuaashishandGitHub c493c1a4ef Update Documentation_issue_template.yaml 2023-02-03 18:08:55 +05:30
kuaashishandGitHub 7661114a52 Update Documentation_issue_template.yaml 2023-02-03 18:07:39 +05:30
Prianka Liz Kariat eeaa011998 Updated documentation of iOS text embedder tests 2023-02-03 18:06:05 +05:30
Prianka Liz Kariat e290f9cf30 Added a note about swift test coverage in iOS text embedder tests 2023-02-03 18:05:49 +05:30
kuaashishandGitHub 1004fb48c7 Update Documentation_issue_template.yaml 2023-02-03 18:04:07 +05:30
kuaashishandGitHub acadf74f6c Update Documentation_issue_template.yaml 2023-02-03 18:03:32 +05:30
kuaashishandGitHub e93650f782 Update Documentation_issue_template.yaml 2023-02-03 18:03:17 +05:30
kuaashishandGitHub e5789396f3 Update Documentation_issue_template.yaml 2023-02-03 18:00:16 +05:30
kuaashishandGitHub 6688b12718 Update Documentation_issue_template.yaml 2023-02-03 17:59:39 +05:30
kuaashishandGitHub 9d55f14bf7 Update Documentation_issue_template.yaml 2023-02-03 17:59:28 +05:30
kuaashishandGitHub a6ad8b521f Update Documentation_issue_template.yaml 2023-02-03 17:57:28 +05:30
kuaashishandGitHub 8550dd86e5 Update Documentation_issue_template.yaml 2023-02-03 17:57:00 +05:30
kuaashishandGitHub ab11f852b0 Update Documentation_issue_template.yaml 2023-02-03 17:55:52 +05:30
kuaashishandGitHub 642922afcb Update Documentation_issue_template.yaml 2023-02-03 17:55:38 +05:30
kuaashishandGitHub 2ced054d81 Update Documentation_issue_template.yaml 2023-02-03 17:53:26 +05:30
kuaashishandGitHub eed7c954f8 Update Documentation_issue_template.yaml 2023-02-03 17:53:09 +05:30
kuaashishandGitHub b0ca0bce4f Update Documentation_issue_template.yaml 2023-02-03 17:51:04 +05:30
kuaashishandGitHub 61515655d1 Create Documentation_issue_template.yaml 2023-02-03 17:49:48 +05:30
kuaashishandGitHub 508a7cbfa5 Update feature_request_issue_template.yaml 2023-02-03 17:32:28 +05:30
kuaashishandGitHub eecfefbcb5 Update feature_request_issue_template.yaml 2023-02-03 17:32:17 +05:30
kuaashishandGitHub 5b595b5f30 Update feature_request_issue_template.yaml 2023-02-03 17:30:07 +05:30
kuaashishandGitHub 9ed873c93b Update feature_request_issue_template.yaml 2023-02-03 17:29:50 +05:30
kuaashishandGitHub fecb4f64fc Update feature_request_issue_template.yaml 2023-02-03 17:26:43 +05:30
kuaashishandGitHub 5c83bdde7a Delete feature_request_issue_template.yaml 2023-02-03 17:25:02 +05:30
kuaashishandGitHub 4071d149f2 Create feature_request_issue_template.yaml 2023-02-03 17:24:28 +05:30
kuaashishandGitHub e055fdb62b Create feature_request_issue_template.yaml 2023-02-03 17:22:34 +05:30
kuaashishandGitHub b129978f89 Update bug_issue_template.yaml 2023-02-03 17:09:24 +05:30
kuaashishandGitHub e29a54221a Update bug_issue_template.yaml 2023-02-03 17:09:14 +05:30
kuaashishandGitHub 810cef8dac Create bug_issue_template.yaml 2023-02-03 17:08:29 +05:30
kuaashishandGitHub 388c746839 Update studio_issue_template.yaml 2023-02-03 16:48:50 +05:30
kuaashishandGitHub 3a10b7c44b Update studio_issue_template.yaml 2023-02-03 16:48:30 +05:30
kuaashishandGitHub c851c6e5dc Update studio_issue_template.yaml 2023-02-03 16:46:13 +05:30
kuaashishandGitHub 0de0035420 Update studio_issue_template.yaml 2023-02-03 16:43:14 +05:30
kuaashishandGitHub 147890fc33 Create studio_issue_template.yaml 2023-02-03 16:42:58 +05:30
kuaashishandGitHub 22e05fc16a Update and rename Solution(Legacy_issue_template.yaml to Solution(Legacy_issue_template).yaml 2023-02-03 16:35:23 +05:30
kuaashishandGitHub 23cf1ee8c3 Create Solution(Legacy_issue_template.yaml 2023-02-03 16:32:03 +05:30
kuaashishandGitHub 14d51d28af Update task_issue_template.yaml 2023-02-03 16:18:54 +05:30
kuaashishandGitHub f197652c3a Create task_issue_template.yaml 2023-02-03 16:16:25 +05:30
kuaashishandGitHub e5a6d3ec3b Update model_maker_issue_template1.yaml 2023-02-03 16:05:33 +05:30
kuaashishandGitHub 4139ce76e0 Update model_maker_issue_template1.yaml 2023-02-03 16:03:01 +05:30
kuaashishandGitHub ca6e4cb7a5 Update model_maker_issue_template1.yaml 2023-02-03 16:01:56 +05:30
kuaashishandGitHub 434c1143ae Update and rename model_maker_issue_template.yaml to model_maker_issue_template1.yaml 2023-02-03 16:01:18 +05:30
kuaashishandGitHub 93ef2f69fb Update model_maker_issue_template.yaml 2023-02-03 16:00:21 +05:30
kuaashishandGitHub d01901f156 Update model_maker_issue_template.yaml 2023-02-03 15:59:15 +05:30
kuaashishandGitHub 79b747969e Update model_maker_issue_template.yaml 2023-02-03 15:58:48 +05:30
kuaashishandGitHub 9de43a83f6 Update model_maker_issue_template.yaml 2023-02-03 15:53:24 +05:30
kuaashishandGitHub 8c60e412b2 Update model_maker_issue_template.yaml 2023-02-03 15:51:37 +05:30
kuaashishandGitHub 96936cf6b2 Update model_maker_issue_template.yaml 2023-02-03 15:50:00 +05:30
kuaashishandGitHub d0486ec9fd Update model_maker_issue_template.yaml 2023-02-03 15:48:18 +05:30
kuaashishandGitHub 79e39e6d52 Update model_maker_issue_template.yaml 2023-02-03 15:47:37 +05:30
kuaashishandGitHub dd10c54ed5 Create model_maker_issue_template.yaml 2023-02-03 15:45:51 +05:30
kuaashishandGitHub 182dfeb4e9 Update build.yaml 2023-02-03 15:25:44 +05:30
kuaashishandGitHub 4ca9b2d43a Update build.yaml 2023-02-03 15:25:23 +05:30
kuaashishandGitHub eea62dca15 Update build.yaml 2023-02-03 15:24:47 +05:30
kuaashishandGitHub 0f7743db1b Update build.yaml 2023-02-03 15:07:33 +05:30
kuaashishandGitHub 2c84077859 Update build.yaml 2023-02-03 15:02:07 +05:30
kuaashishandGitHub 1b0923c3d7 Delete 00-build-installation-issue.md 2023-02-03 14:59:10 +05:30
kuaashishandGitHub d97df4071e Merge branch 'google:master' into master 2023-02-03 14:58:34 +05:30
kuaashishandGitHub f7b035b15e Update build.yaml 2023-02-03 14:57:28 +05:30
kuaashishandGitHub be42a73f43 Update build.yaml 2023-02-03 14:56:44 +05:30
kuaashishandGitHub 6a90f8b27e Update build.yaml 2023-02-03 14:55:36 +05:30
kuaashishandGitHub d554f92d71 Update build.yaml 2023-02-03 14:53:50 +05:30
kuaashishandGitHub 78502bb5fd Update build.yaml 2023-02-03 14:49:37 +05:30
kuaashishandGitHub daa2116773 Update build.yaml 2023-02-03 14:48:52 +05:30
kuaashishandGitHub 43a97637db Update build.yaml 2023-02-03 14:47:37 +05:30
kuaashishandGitHub 598624d201 Update build.yaml 2023-02-03 14:43:29 +05:30
kuaashishandGitHub b75c7dedfc Update build.yaml 2023-02-03 14:41:13 +05:30
kuaashishandGitHub 01a740db5f Update build.yaml 2023-02-03 14:36:16 +05:30
kuaashishandGitHub 76685b2213 Update build.yaml 2023-02-03 14:35:05 +05:30
kuaashishandGitHub a235621b16 Update build.yaml 2023-02-03 14:34:41 +05:30
kuaashishandGitHub 90c5dc19d1 Update build.yaml 2023-02-03 14:33:47 +05:30
kuaashishandGitHub 9af30f98a2 Create build.yaml 2023-02-03 14:32:38 +05:30
Prianka Liz Kariat 3b55fb9f6a Added iOS test for quantized embedding 2023-02-03 13:42:32 +05:30
Prianka Liz Kariat b5b10e7681 Added iOS test for different themes in text embedder 2023-02-03 13:10:13 +05:30
Prianka Liz Kariat a512e6b5f5 Updated MPPTextEmbedder Documentation 2023-02-03 12:49:00 +05:30
Prianka Liz Kariat 6ca1efdd55 Updated MPPTextEmbedder Documentation 2023-02-03 12:48:06 +05:30
kuaashishandGitHub 1a26071a41 Update install.md
This will fix the Subsystem typo in line 654
2023-02-03 12:12:45 +05:30
MediaPipe TeamandCopybara-Service e485961c2d fixes spelling mistake
PiperOrigin-RevId: 506697863
2023-02-02 12:49:30 -08:00
Prianka Liz Kariat 20002f191a Changed documentation 2023-02-02 18:38:19 +05:30
Prianka Liz Kariat d625918995 Added swift tests for text embedder 2023-02-02 18:36:55 +05:30
Prianka Liz Kariat 474e994a5f Added text embedder objective c tests 2023-02-02 17:30:05 +05:30
Prianka Liz Kariat 867520af1c Added cosine similarity to MPPTextEmbedder 2023-02-02 17:29:51 +05:30
Prianka Liz Kariat 84e1c93ffb Added MPPCosineSimilarity 2023-02-02 17:22:56 +05:30
kuaashishandGitHub c79350da5f Update javascript.md
This will fix the typo from "Supported plaforms" to "Supported platforms" in line 32
2023-02-02 17:12:09 +05:30
Sebastian SchmidtandCopybara-Service 286dde97ad Make TextEmbedder and TextClassifier tests pass on Windows
PiperOrigin-RevId: 506421383
2023-02-01 14:22:04 -08:00
Copybara-Service bdd77b0d61 Merge pull request #4048 from priankakariatyml:ios-text-embedder
PiperOrigin-RevId: 506407904
2023-02-01 13:32:58 -08:00
Sebastian SchmidtandCopybara-Service 0f3cf9c56a Add "noasan" to MPPTextClassifierObjcTest
PiperOrigin-RevId: 506366650
2023-02-01 11:06:17 -08:00
Sebastian SchmidtandCopybara-Service 83e33b4dbe Internal change
PiperOrigin-RevId: 506345436
2023-02-01 09:55:19 -08:00
Juhyun LeeandCopybara-Service 0919a6c0a3 Log which InferenceCalculator impl is used at runtime.
PiperOrigin-RevId: 506343086
2023-02-01 09:45:36 -08:00
MediaPipe TeamandCopybara-Service 3ee377f671 Internal change
PiperOrigin-RevId: 506312863
2023-02-01 07:42:20 -08:00
Prianka Liz Kariat bd507b2d7b Updated MPPEmbeddingHelpers to reflect type change of quantized embeddings 2023-02-01 19:27:05 +05:30
Prianka Liz Kariat cd1cb87ff6 Merge branch 'master' into ios-text-embedder 2023-02-01 19:11:01 +05:30
Prianka Liz Kariat 85c310d01c Updated formatting 2023-02-01 18:52:55 +05:30
Prianka Liz Kariat d588f73a6d Added MPPTextEmbedder 2023-02-01 18:51:30 +05:30
Prianka Liz Kariat 38eac174e9 Added MPPTextEmbedderResultHelpers 2023-02-01 18:51:01 +05:30
Prianka Liz Kariat 42e712e911 Added MPPTextEmbedderOptionsHelpers 2023-02-01 18:50:14 +05:30
Prianka Liz Kariat 69809e2181 Updated formatting 2023-02-01 18:48:10 +05:30
Prianka Liz Kariat ffc9f1d47e Added MPPEmbeddingResultHelpers 2023-02-01 18:47:23 +05:30
Prianka Liz Kariat 24bd104b0f Added MPPEmbedding Helpers 2023-02-01 18:46:11 +05:30
MediaPipe TeamandCopybara-Service d283e6a05a Support downloading model files on-demand from GCS in model_maker
PiperOrigin-RevId: 506174708
2023-01-31 18:43:35 -08:00
MediaPipe TeamandCopybara-Service b53acf6267 Internal change
PiperOrigin-RevId: 506059384
2023-01-31 11:20:13 -08:00
MediaPipe TeamandCopybara-Service 5730dec260 Internal change
PiperOrigin-RevId: 506053206
2023-01-31 11:03:41 -08:00
Copybara-Service 0863a8a1e7 Merge pull request #4030 from priankakariatyml:ios-text-embedder
PiperOrigin-RevId: 506048365
2023-01-31 10:47:19 -08:00
MediaPipe TeamandCopybara-Service 591eb204a6 Internal change
PiperOrigin-RevId: 506027661
2023-01-31 09:36:56 -08:00
Sebastian SchmidtandCopybara-Service be3bddc620 Add Text Embedder tests for text with different themes
PiperOrigin-RevId: 506023265
2023-01-31 09:23:30 -08:00
Jiuqiang TangandCopybara-Service 2c4dece023 Internal change
PiperOrigin-RevId: 505723714
2023-01-30 10:00:22 -08:00
MediaPipe TeamandCopybara-Service f9f6acffed Make NORM_RECT optional for GestureRecognizerGraph and add PALM_DETECTION output PORT
PiperOrigin-RevId: 505712542
2023-01-30 09:17:38 -08:00
Prianka Liz Kariat 8c21dc02a6 Updated to types of float and quantized embedding 2023-01-30 11:42:33 +05:30
MediaPipe TeamandCopybara-Service ee2f940e1f Make TensorToVectorFloatCalculator compatible with unaligned tensors.
No performance impact is expected, since the unaligned Eigen::TensorMap is used only to populate a std::vector<float>.

PiperOrigin-RevId: 505251810
2023-01-27 18:08:03 -08:00
Sebastian SchmidtandCopybara-Service 702cc0c42c Change documentation to use shallow clones of the MP Repo
PiperOrigin-RevId: 505234066
2023-01-27 16:21:34 -08:00
Sebastian SchmidtandCopybara-Service dc3fdf6eb4 Internal change
PiperOrigin-RevId: 505193224
2023-01-27 13:16:06 -08:00
MediaPipe TeamandCopybara-Service a6f6be9512 Fix incorrect uint8 -> int8 conversion in JS cosine similarity.
PiperOrigin-RevId: 505135368
2023-01-27 09:21:58 -08:00
MediaPipe TeamandCopybara-Service 1df4511e9d Add YuvImage as a GpuBuffer storage backend.
PiperOrigin-RevId: 505128789
2023-01-27 08:51:31 -08:00
MediaPipe TeamandCopybara-Service e059d55d29 Correctly check refCount in finalize.
PiperOrigin-RevId: 505057866
2023-01-27 01:52:48 -08:00
Prianka Liz Kariat 8531803462 Updated documentation of embedding containers 2023-01-27 11:08:39 +05:30
Sebastian SchmidtandCopybara-Service c29ab7f083 Internal change
PiperOrigin-RevId: 504928797
2023-01-26 13:36:45 -08:00
Sebastian SchmidtandCopybara-Service 4d38557f11 Add MediaPipe Image Segmenter task for Web
PiperOrigin-RevId: 504912518
2023-01-26 12:31:54 -08:00
MediaPipe TeamandCopybara-Service 29001234d5 Replace SourceOrNodeOutput with Source.
PiperOrigin-RevId: 504883990
2023-01-26 10:46:12 -08:00
MediaPipe TeamandCopybara-Service 2547f07c77 Add FrameBuffer format.
PiperOrigin-RevId: 504838580
2023-01-26 07:40:52 -08:00
Jiuqiang TangandCopybara-Service 0566e0e7ca Fix the output stream tag of the end loop calculator in the example code.
PiperOrigin-RevId: 504708273
2023-01-25 17:52:55 -08:00
MediaPipe TeamandCopybara-Service be546d22fc Update test to reflect the recommended graph construction style:
First, graph inputs and their names:
  - Makes it clear what inputs graph has
  - Indirectly demands for type specification e.g. Stream<AnyType> a = graph.In(0); vs Stream<int> a = graph.In(0).Cast<int>();
Then graph nodes
  - Nodes are added and used as they needed
  - One node is not mixed in other nodes, only its outputs
  - Indirectly demands for type specification e.g. Stream<AnyType> a = node.Out(0); vs Stream<int> a = node.Out(0).Cast<int>();
Then graph outputs
  - Makes it clear what outputs graph has

The recommended structure keep C++ graph similar to pbtxt representation.

PiperOrigin-RevId: 504701023
2023-01-25 17:13:17 -08:00
MediaPipe TeamandCopybara-Service ff0ccfc209 Internal change
PiperOrigin-RevId: 504677663
2023-01-25 15:31:21 -08:00
Sebastian SchmidtandCopybara-Service 1538740dcb Formatting fix
PiperOrigin-RevId: 504599712
2023-01-25 10:33:09 -08:00
Prianka Liz Kariat 61f7739ff6 Updated documentation 2023-01-25 20:20:22 +05:30
Prianka Liz Kariat d01f75a295 Added iOS text embedder result files 2023-01-25 20:19:57 +05:30
Prianka Liz Kariat 168ea0a9ea Added MPPTextEmbedderResult 2023-01-25 20:19:40 +05:30
Prianka Liz Kariat 60e72bf165 Added MPPTextEmbedderOptions 2023-01-25 20:19:27 +05:30
Prianka Liz Kariat db5ee6689f Added MPPEmbeddingResult 2023-01-25 20:17:04 +05:30
Prianka Liz Kariat 7d62402768 Added MPPEmbedding 2023-01-25 20:16:54 +05:30
MediaPipe TeamandCopybara-Service afb0182935 Internal model maker change.
PiperOrigin-RevId: 504472342
2023-01-24 23:16:21 -08:00
Sebastian SchmidtandCopybara-Service 5dc81c4c27 Remove unused import on strings.h
PiperOrigin-RevId: 504397437
2023-01-24 15:56:15 -08:00
Yuqi LiandCopybara-Service 679dbb3fd8 nit: update the metadata_schema.fbs file path.
PiperOrigin-RevId: 504380873
2023-01-24 14:46:25 -08:00
Sebastian SchmidtandCopybara-Service ce9fec806c Internal change
PiperOrigin-RevId: 504341886
2023-01-24 12:18:15 -08:00
Mike KruskalandCopybara-Service 9cde57d830 Internal change
PiperOrigin-RevId: 504341832
2023-01-24 12:14:02 -08:00
MediaPipe TeamandCopybara-Service 4e135ccdb9 Internal Model Maker change.
PiperOrigin-RevId: 504315641
2023-01-24 10:38:25 -08:00
MediaPipe TeamandCopybara-Service 2465e47b01 Stream/SidePacket == and != operators
PiperOrigin-RevId: 504114182
2023-01-23 16:43:23 -08:00
MediaPipe TeamandCopybara-Service 873d7181bf Add mediapipe tasks face detector graph
PiperOrigin-RevId: 504078951
2023-01-23 14:15:21 -08:00
Sebastian SchmidtandCopybara-Service ccd1461add Don't error in ExternalFile handler on Windows if FileContent is provided
PiperOrigin-RevId: 504069137
2023-01-23 13:38:41 -08:00
MediaPipe TeamandCopybara-Service 69d354fc89 Use c++ struct as hand landmark detection results.
PiperOrigin-RevId: 504048095
2023-01-23 12:12:09 -08:00
Nikolay ChirkovandCopybara-Service 1124569c29 Tensor: Make tensor not requiring "-x objective-c++" option.
In this case tensor.h is compiled differently for C++ and Objective-C++ that violates ODR (once definition rule). Tensor has no virtual methods conditionally compiled but some Metal-related data members. Instead, unique_ptr to MtlResources that is declared as forward structure is unconditionally defined in the tensor class. MtlResources is defined differently in cc-file only that compiled just once per project so no ODR violation is here.

PiperOrigin-RevId: 504029286
2023-01-23 11:00:35 -08:00
MediaPipe TeamandCopybara-Service 921b6a6bef This CL will fix the typo from _PALM_LANMARKS to _PALM_LANDMARKS.
PiperOrigin-RevId: 503352055
2023-01-19 22:14:16 -08:00
Copybara-Service 4b9a52dc34 Merge pull request #3995 from priankakariatyml:ios-text-classifier-tests
PiperOrigin-RevId: 503242486
2023-01-19 12:59:49 -08:00
Sebastian SchmidtandCopybara-Service db1a89324e Add mediapipe::Image output to the graph runner
PiperOrigin-RevId: 503204918
2023-01-19 10:41:13 -08:00
Sebastian SchmidtandCopybara-Service a02097ea08 Fix comments
PiperOrigin-RevId: 503195768
2023-01-19 10:08:44 -08:00
MediaPipe TeamandCopybara-Service dcd2adad53 Removing broken links. They might not be relevant since we only support TfLite models.
PiperOrigin-RevId: 503183358
2023-01-19 09:19:48 -08:00
Adam CozzetteandCopybara-Service 7a7cc77a81 Internal change
PiperOrigin-RevId: 503157344
2023-01-19 07:20:04 -08:00
MediaPipe TeamandCopybara-Service e2dedcbfe5 Add SQRT_HANN window type to both SpectrogramCalculator and InverseSpectrogramCalculator.
PiperOrigin-RevId: 503041493
2023-01-18 19:42:13 -08:00
Hadon NashandCopybara-Service 97af47ebf5 Internal change
PiperOrigin-RevId: 503035081
2023-01-18 18:53:42 -08:00
Hadon NashandCopybara-Service 66634bbef8 Internal change
PiperOrigin-RevId: 503011674
2023-01-18 16:38:11 -08:00
MediaPipe TeamandCopybara-Service e56fa8f258 Source/SideSource -> Stream/SidePacket
PiperOrigin-RevId: 502923931
2023-01-18 11:02:41 -08:00
Nikolay ChirkovandCopybara-Service 5687d19dec Tensor: remove unused and unimplemented SetPreferredStorageType methods.
PiperOrigin-RevId: 502893019
2023-01-18 09:09:20 -08:00
Jiuqiang TangandCopybara-Service 29484702ce Add process_timestamp_bounds into RectToRenderScaleCalculatorOptions.
PiperOrigin-RevId: 502877541
2023-01-18 08:02:47 -08:00
MediaPipe TeamandCopybara-Service 3688757d17 Fix load_metadata_buffer for empty metadata
PiperOrigin-RevId: 502870428
2023-01-18 07:28:42 -08:00
Sebastian SchmidtandCopybara-Service e484bd681e Export all input and output types
PiperOrigin-RevId: 502764544
2023-01-17 20:57:16 -08:00
Sebastian SchmidtandCopybara-Service d5e60eb658 Internal change
PiperOrigin-RevId: 502764352
2023-01-17 20:53:34 -08:00
Sebastian SchmidtandCopybara-Service 0b97c6e67d Update the MP Wasm builds to latest version.
PiperOrigin-RevId: 502709126
2023-01-17 15:54:41 -08:00
Camillo LugaresiandCopybara-Service 7894c92ab7 Internal change
PiperOrigin-RevId: 502709070
2023-01-17 15:51:07 -08:00
Sebastian SchmidtandCopybara-Service 088249eb36 Export all input and output types
PiperOrigin-RevId: 502649430
2023-01-17 11:59:33 -08:00
Jiuqiang TangandCopybara-Service 7a4b450c50 Resolve the error "call to 'abs' is ambiguous".
PiperOrigin-RevId: 502630518
2023-01-17 10:53:14 -08:00
Jiuqiang TangandCopybara-Service 7974171c3d Merge classificationResultList() and classificationResult() to be classificationResults(), and similar for embeddingResults().
PiperOrigin-RevId: 502601043
2023-01-17 09:06:49 -08:00
Camillo LugaresiandCopybara-Service c1f5920ecf Add web performance tracing to the MEDIAPIPE_PROFILING repertoire
This records the MEDIAPIPE_PROFILING tracing annotations to the browser's trace using the user timing API. See https://developer.mozilla.org/en-US/docs/Web/API/User_Timing_API

To enable, build with --define MEDIAPIPE_WEB_PROFILING=1 --define DRISHTI_PROFILING=1

PiperOrigin-RevId: 502422030
2023-01-16 12:59:18 -08:00
MediaPipe TeamandCopybara-Service ffd8486d0d Add a stub WriteProfile method to GraphProfilerStub.
PiperOrigin-RevId: 502388455
2023-01-16 08:38:05 -08:00
Prianka Liz Kariat 67735a6fd3 Added category indices in iOS failure description 2023-01-16 14:01:10 +05:30
Prianka Liz Kariat cf945d3aeb Removed unused variable 2023-01-16 13:59:51 +05:30
Prianka Liz Kariat a0b3e620e4 Removed unused methods 2023-01-16 13:12:27 +05:30
Prianka Liz Kariat f7fc8a6eca Updated method names in tests 2023-01-16 13:05:29 +05:30
Prianka Liz Kariat 8ecf77f760 Updated comment style in methods 2023-01-16 13:02:33 +05:30
Prianka Liz Kariat 30533be321 Reformatted comments 2023-01-16 13:00:10 +05:30
MediaPipe TeamandCopybara-Service 92a2e02ace Internal change
PiperOrigin-RevId: 501971410
2023-01-13 17:05:19 -08:00
Sebastian SchmidtandCopybara-Service aef4cca406 Copy README.md to NPM package root
PiperOrigin-RevId: 501929871
2023-01-13 13:48:02 -08:00
Sebastian SchmidtandCopybara-Service f997c0ab1a Reject RegionOfInterest in not supported tasks
PiperOrigin-RevId: 501872455
2023-01-13 09:53:57 -08:00
MediaPipe TeamandCopybara-Service 69757d7924 Internal change
PiperOrigin-RevId: 501862194
2023-01-13 09:05:48 -08:00
Prianka Liz Kariat 95f9f0fb88 Updated formatting 2023-01-13 21:18:10 +05:30
Prianka Liz Kariat c4c07acc1e Updated comments of MPPCommonUtils 2023-01-13 21:18:01 +05:30
Prianka Liz Kariat 2a53d78ae4 Added swift and objective tests for iOS text classifier 2023-01-13 21:05:44 +05:30
Prianka Liz Kariat 9e0b85c9b5 Added module name for iOS text classifier 2023-01-13 21:05:17 +05:30
Prianka Liz Kariat c40356c628 Added ios.bzl 2023-01-13 21:04:56 +05:30
Prianka Liz Kariat 0a707256e3 Updates to method signatures of iOS text classifier 2023-01-13 21:04:43 +05:30
Prianka Liz Kariat fa30100059 Changed swift name of MPPCategory 2023-01-13 21:04:17 +05:30
Prianka Liz Kariat 5642980ab0 Updated iOS error implementation to mimic java 2023-01-13 21:04:03 +05:30
Copybara-Service 4e19a5a0dc Merge pull request #3988 from priankakariatyml:ios-text-classifier-impl
PiperOrigin-RevId: 501739316
2023-01-12 20:25:02 -08:00
Jiuqiang TangandCopybara-Service 8156da3418 ClassificationAggregationCalculator should fill in the timestamp_ms field of the classification results in the stream mode.
Per user feedback, the consistency between the packet timestamp and the timestamp field of the classification result helps reducing the confusion.

PiperOrigin-RevId: 501657922
2023-01-12 13:54:55 -08:00
MediaPipe TeamandCopybara-Service 1683d572ed Internal change
PiperOrigin-RevId: 501600938
2023-01-12 10:22:00 -08:00
MediaPipe TeamandCopybara-Service 74b60780c7 Internal change
PiperOrigin-RevId: 501594400
2023-01-12 10:00:44 -08:00
Jiuqiang TangandCopybara-Service 5c74ed2ae5 EmbeddingAggregationCalculator should fill in the timestamp_ms field of the embedding results in the stream mode.
Per user feedback, the consistency between the packet timestamp and the timestamp field of the embedding result helps reducing the confusion.

PiperOrigin-RevId: 501572379
2023-01-12 08:30:07 -08:00
MediaPipe TeamandCopybara-Service 9cbb76939d Adds smaller MobileBERT model.
PiperOrigin-RevId: 501451414
2023-01-11 20:35:39 -08:00
Nikolay ChirkovandCopybara-Service 8830eefa0b Internal change.
PiperOrigin-RevId: 501403332
2023-01-11 16:07:22 -08:00
MediaPipe TeamandCopybara-Service 36be94f861 Internal change
PiperOrigin-RevId: 501378130
2023-01-11 14:24:00 -08:00
MediaPipe TeamandCopybara-Service 5612af68cd Propagate compatible_with for drishti_proto_library
PiperOrigin-RevId: 501356895
2023-01-11 13:01:57 -08:00
MediaPipe TeamandCopybara-Service 0e56bd38f3 Fix for CHECK failure due to pointer description sometimes being larger than allocated string space
PiperOrigin-RevId: 501355568
2023-01-11 12:58:37 -08:00
Prianka Liz Kariat c7e36f8720 Re-ordered dependencies in build file 2023-01-11 20:31:46 +05:30
Prianka Liz Kariat fe05a8d201 Reformatted code 2023-01-11 20:24:17 +05:30
Prianka Liz Kariat b1ded2f700 Added iOS text classifier 2023-01-11 20:22:33 +05:30
Prianka Liz Kariat a0220de233 Added iOS text classifier result helpers 2023-01-11 20:22:20 +05:30
Prianka Liz Kariat 54161cc1ab Added iOS text classifier options helpers 2023-01-11 20:22:02 +05:30
Prianka Liz Kariat 1161ebce9d Added iOS text classifier result 2023-01-11 16:22:09 +05:30
Prianka Liz Kariat ed6abbbe43 Added iOS text classifier options 2023-01-11 16:21:28 +05:30
Nikolay ChirkovandCopybara-Service 54268594dd Internal change.
PiperOrigin-RevId: 501136760
2023-01-10 17:38:13 -08:00
Copybara-Service e869e57cb4 Merge pull request #3975 from priankakariatyml:ios-base-options-helpers
PiperOrigin-RevId: 501073613
2023-01-10 13:07:47 -08:00
Jiuqiang TangandCopybara-Service 25abd122b3 Support AudioRecord in MediaPipe audio tasks in Java.
PiperOrigin-RevId: 501019327
2023-01-10 09:45:40 -08:00
Copybara-Service b747fc481b Merge pull request #3973 from priankakariatyml:ios-task-classifier-result
PiperOrigin-RevId: 501005242
2023-01-10 08:47:45 -08:00
Jiuqiang TangandCopybara-Service 6032604f94 Hide base task api classes for MediaPipe Tasks Python from API docs
PiperOrigin-RevId: 501004802
2023-01-10 08:43:51 -08:00
Sebastian SchmidtandCopybara-Service d7ee875356 Fix spacing issue in test name
PiperOrigin-RevId: 500833769
2023-01-09 16:20:08 -08:00
Liam Miller-CushonandCopybara-Service 76a7c9d5d4 Internal change
PiperOrigin-RevId: 500813290
2023-01-09 14:49:45 -08:00
MediaPipe TeamandCopybara-Service 704964be33 Fix accidental suppressions of GLSL linker error reporting
PiperOrigin-RevId: 500802177
2023-01-09 14:05:47 -08:00
Sebastian SchmidtandCopybara-Service 0831023114 Use uppercase enum constants for RunningMode
PiperOrigin-RevId: 500760402
2023-01-09 11:11:21 -08:00
MediaPipe TeamandCopybara-Service d40fa6b16d Internal Model Maker change.
PiperOrigin-RevId: 500758488
2023-01-09 11:04:45 -08:00
Sebastian SchmidtandCopybara-Service 73f4636292 Create README.md files to NPM packages
PiperOrigin-RevId: 500750516
2023-01-09 10:36:49 -08:00
MediaPipe TeamandCopybara-Service c6cf598774 Minor fix for max_queue_size documentation
PiperOrigin-RevId: 500738798
2023-01-09 09:54:09 -08:00
Sebastian SchmidtandCopybara-Service 2b9299959c Internal change
PiperOrigin-RevId: 500730237
2023-01-09 09:22:02 -08:00
Sebastian SchmidtandCopybara-Service 1bbe065647 Simplify default options for GestureRecognize
PiperOrigin-RevId: 500729643
2023-01-09 09:18:08 -08:00
Nikolay ChirkovandCopybara-Service e0a254789a Internal change.
PiperOrigin-RevId: 500331015
2023-01-06 22:15:18 -08:00
Sebastian SchmidtandCopybara-Service 7f043b7de1 Allow split_vector_calculator to be build with iOS and MEDIAPIPE_DISABLE_GPU
PiperOrigin-RevId: 500327774
2023-01-06 21:45:15 -08:00
Sebastian SchmidtandCopybara-Service c9ebc6fa60 Use synthetic timestamps in Web when none provided
PiperOrigin-RevId: 500327275
2023-01-06 21:41:30 -08:00
Sebastian SchmidtandCopybara-Service ed0054836a Allow task to recover after a failed graph start
PiperOrigin-RevId: 500324587
2023-01-06 21:06:47 -08:00
Sebastian SchmidtandCopybara-Service b4ede6db7b Fix typo in Category.java
PiperOrigin-RevId: 500324008
2023-01-06 21:02:16 -08:00
Sebastian SchmidtandCopybara-Service 9055effddd Add ImageProcessingOptions to all Vision Tasks
PiperOrigin-RevId: 500323261
2023-01-06 20:57:39 -08:00
Copybara-Service e11ba95adf Merge pull request #3974 from priankakariatyml:ios-task-updates
PiperOrigin-RevId: 500323180
2023-01-06 20:53:33 -08:00
Sebastian SchmidtandCopybara-Service 9b34a105cf Do not depend on Image methods in TaskRunner
PiperOrigin-RevId: 500299571
2023-01-06 18:20:36 -08:00
MediaPipe TeamandCopybara-Service 2cce88080e Internal change
PiperOrigin-RevId: 500271109
2023-01-06 15:29:19 -08:00
Prianka Liz Kariat 14e3de49ad Added MPPTextTaskRunner 2023-01-06 16:37:31 +05:30
Prianka Liz Kariat b91b485035 Added MPPBaseOptions Helpers 2023-01-06 16:36:28 +05:30
Prianka Liz Kariat b6bcc35ade Added provision for packets callback in iOS task runner 2023-01-06 16:36:15 +05:30
Prianka Liz Kariat c6bae99a2f Updated formatting in MPPTextPacketCreator.mm 2023-01-06 16:25:56 +05:30
Prianka Liz Kariat bc1b069edf Updated property name in MPPTaskResult 2023-01-06 16:24:41 +05:30
Prianka Liz Kariat 16f9831c3f Updated formatting in MPPTaskOptions.m 2023-01-06 16:23:37 +05:30
Prianka Liz Kariat 61d16b284b Updated comments in MPPTaskOptions.h 2023-01-06 16:23:22 +05:30
Prianka Liz Kariat 27ce2ec00f Updated C++ types to camel case in MPPTaskInfo 2023-01-06 16:22:11 +05:30
Prianka Liz Kariat f37689fc33 Updated documentation for MPPCommonUtils.m 2023-01-06 16:15:53 +05:30
Prianka Liz Kariat 4e38c7e623 Updated documentation for MPPCommon.h 2023-01-06 16:15:32 +05:30
Prianka Liz Kariat 8f74a175d8 Removed MPPClassifierOptions and helpers 2023-01-06 16:08:06 +05:30
Prianka Liz Kariat 89aad67a87 Added iOS helpers for classification result containers 2023-01-06 16:07:50 +05:30
Prianka Liz Kariat 33df6c042f Added iOS result containers for classification tasks 2023-01-06 16:07:11 +05:30
Sebastian SchmidtandCopybara-Service 667fd81ddc Internal change
PiperOrigin-RevId: 499956657
2023-01-05 11:43:43 -08:00
Sebastian SchmidtandCopybara-Service 81a46bb31a Internal change
PiperOrigin-RevId: 499902323
2023-01-05 09:14:07 -08:00
Nikolay ChirkovandCopybara-Service 35293d88bc Tensor: move into tensor sub-directory.
PiperOrigin-RevId: 499896489
2023-01-05 08:56:37 -08:00
MediaPipe TeamandCopybara-Service 463cbb60ee Fix RGBA vs RGB selection when creating GLTexture.
PiperOrigin-RevId: 499877590
2023-01-05 07:58:07 -08:00
MediaPipe TeamandCopybara-Service 43bf02443c Option to remove overlapping values computed for different timestamps.
PiperOrigin-RevId: 499635143
2023-01-04 17:35:25 -08:00
MediaPipe TeamandCopybara-Service 24cc0672c4 Internal change
PiperOrigin-RevId: 499529022
2023-01-04 10:59:35 -08:00
MediaPipe TeamandCopybara-Service e3131d7d78 Internal change
PiperOrigin-RevId: 499521620
2023-01-04 10:33:04 -08:00
Nikolay ChirkovandCopybara-Service 9a70af1464 Internal change.
PiperOrigin-RevId: 499496793
2023-01-04 08:54:29 -08:00
Nikolay ChirkovandCopybara-Service a4ea606eac Internal change.
PiperOrigin-RevId: 499490514
2023-01-04 08:23:56 -08:00
MediaPipe TeamandCopybara-Service add5600d0d Internal change
PiperOrigin-RevId: 499351795
2023-01-03 17:20:46 -08:00
MediaPipe TeamandCopybara-Service e7dc989f71 Internal Change
PiperOrigin-RevId: 499313491
2023-01-03 14:14:21 -08:00
Copybara-Service a7bb0aba8f Merge pull request #3937 from priankakariatyml:ios-task-files
PiperOrigin-RevId: 499296662
2023-01-03 13:04:55 -08:00
Sebastian SchmidtandCopybara-Service 75b87e0e32 Internal change
PiperOrigin-RevId: 499283559
2023-01-03 12:12:24 -08:00
Sebastian SchmidtandCopybara-Service 68f247a5c7 Internal change
PiperOrigin-RevId: 499282085
2023-01-03 12:05:43 -08:00
Sebastian SchmidtandCopybara-Service 987f4dc1ed Make addJsamineCustomFloatEqualityTest configurable
PiperOrigin-RevId: 499263931
2023-01-03 10:55:58 -08:00
MediaPipe TeamandCopybara-Service f53c0eacee Extend tag conversion behavior to also convert : (in addition to the
current `/`, `-`, and `.`) to `_`.

PiperOrigin-RevId: 499243005
2023-01-03 09:41:45 -08:00
Camillo LugaresiandCopybara-Service 2f4bb5d545 Use utility framebuffer in ViewDoneWritingSimulatorWorkaround
This code needs a FBO to bind the texture.

Fixes invalid results when running under simulator.

PiperOrigin-RevId: 499241867
2023-01-03 09:38:19 -08:00
Sebastian SchmidtandCopybara-Service 9252a025e5 Use custom gesture options in GestureRecognizer
PiperOrigin-RevId: 498567432
2022-12-30 05:01:02 -08:00
MediaPipe TeamandCopybara-Service 60c6b155f6 Save an integer id in graph profiler objects to distinguish between different profiler instances during benchmarking.
PiperOrigin-RevId: 498409363
2022-12-29 10:18:10 -08:00
MediaPipe TeamandCopybara-Service aaa16eca1f Sets the graph service packets before initializing (and validating the graph) in the objc graph wrapper.
PiperOrigin-RevId: 498393761
2022-12-29 08:36:03 -08:00
Nikolay ChirkovandCopybara-Service 2d9a969d10 Tensor1: memorize size_alignment when tracking the ahwb usage.
When CPU/GPU buffer allocated and the tracker selects Ahwb storage to be used then the properly recorded alignment must be used.

PiperOrigin-RevId: 498264759
2022-12-28 16:09:02 -08:00
Nikolay ChirkovandCopybara-Service 1924f1cdff Tensor: Fix use_ahwb_ flag and tests on local device involved.
PiperOrigin-RevId: 498249332
2022-12-28 14:29:43 -08:00
Sebastian SchmidtandCopybara-Service 9580f04571 Apply most graph options synchronously
PiperOrigin-RevId: 498244085
2022-12-28 13:58:46 -08:00
MediaPipe TeamandCopybara-Service 7e36a5e2ae Set filecmp.cmp(shallow=False) in model_maker unit tests.
PiperOrigin-RevId: 498218578
2022-12-28 11:25:28 -08:00
Sebastian SchmidtandCopybara-Service 175aff9be8 Update list of issue assignments
PiperOrigin-RevId: 498003950
2022-12-27 11:27:58 -08:00
MediaPipe TeamandCopybara-Service 5a5ff5393a Internal change
PiperOrigin-RevId: 497269082
2022-12-27 10:50:41 -08:00
Hadon NashandCopybara-Service 557cd050f3 Deprecating RealTimeFlowLimiterCalculator in favor of FlowLimiterCalculator.
PiperOrigin-RevId: 497260577
2022-12-22 16:29:14 -08:00
MediaPipe TeamandCopybara-Service 5a71b551e5 Remove duplicate and non-public api for model_maker
PiperOrigin-RevId: 497251246
2022-12-22 15:31:12 -08:00
MediaPipe TeamandCopybara-Service 36f054dfbe Internal model maker change
PiperOrigin-RevId: 497196512
2022-12-22 10:43:15 -08:00
MediaPipe TeamandCopybara-Service 5b90afda70 Internal change
PiperOrigin-RevId: 497191969
2022-12-22 10:23:34 -08:00
MediaPipe TeamandCopybara-Service 9673841605 Internal visibility update
PiperOrigin-RevId: 497185157
2022-12-22 09:51:52 -08:00
Prianka Liz Kariat 48eeae4d9d Formatted code 2022-12-22 12:16:43 +05:30
Prianka Liz Kariat 613ed58890 Inverted condition check in MPPTaskInfo 2022-12-22 12:16:33 +05:30
Prianka Liz Kariat e47256ae55 Clearing proto before assigining new values in MPPClassifierOptions Helpers 2022-12-22 12:10:23 +05:30
MediaPipe TeamandCopybara-Service 69b6d9d970 Internal change
PiperOrigin-RevId: 497043596
2022-12-21 17:44:36 -08:00
Prianka Liz Kariat e559613b9d Updated comments in MPPClassifierOptions.h 2022-12-22 02:02:48 +05:30
Prianka Liz Kariat b4a7644428 Updated comments 2022-12-22 02:01:04 +05:30
Prianka Liz Kariat 20c3388ab6 Updated category allowlist and denylist names 2022-12-22 01:59:38 +05:30
Prianka Liz Kariat c185dc9ad7 Renamed label to category in classifier options 2022-12-22 01:19:01 +05:30
Prianka Liz Kariat e1dfcf03cf Updated comments in MPPClassifierOptions.h 2022-12-22 01:12:34 +05:30
Prianka Liz Kariat 66ee8d47c0 Resorted options 2022-12-22 01:10:07 +05:30
Prianka Liz Kariat 673b38dfe8 Updated comments 2022-12-22 01:08:13 +05:30
Prianka Liz Kariat 54d36dfeda Update MPPClassifierOptions.h 2022-12-22 01:05:50 +05:30
Prianka Liz Kariat 7ae4b7e639 Updated error domain 2022-12-22 01:05:01 +05:30
Prianka Liz Kariat 99c11ff974 Updated comments 2022-12-22 01:03:39 +05:30
Prianka Liz Kariat 1de3694175 Updated comments 2022-12-22 01:02:07 +05:30
Prianka Liz Kariat 1491b3f5a2 Updated comments 2022-12-22 01:00:36 +05:30
Prianka Liz Kariat 20f2e136c5 Updated empty spaces 2022-12-22 00:59:22 +05:30
Prianka Liz Kariat 2943d1668e Updated comments 2022-12-22 00:51:20 +05:30
Prianka Liz Kariat 481f4e960e Updated comments 2022-12-22 00:49:44 +05:30
Prianka Liz Kariat ae28948ca1 Marked designated initializers 2022-12-22 00:49:24 +05:30
Jiuqiang TangandCopybara-Service c8b8d1fe6b Remove scripts for building MediaPipe Python 3.7 wheels.
PiperOrigin-RevId: 496962729
2022-12-21 11:12:27 -08:00
Jiuqiang TangandCopybara-Service 714a6e555b Enable creating mediapipe image c++ packet directly from an Android media image object when its format is RGBA_8888.
PiperOrigin-RevId: 496923491
2022-12-21 08:07:53 -08:00
MediaPipe TeamandCopybara-Service 1341720d6d Internal change
PiperOrigin-RevId: 496854337
2022-12-21 00:55:32 -08:00
MediaPipe TeamandCopybara-Service 5c0f548f5f Switches to tf.keras.optimizers.experimental.AdamW instead of the legacy AdamW.
PiperOrigin-RevId: 496821354
2022-12-20 20:53:13 -08:00
Jiuqiang TangandCopybara-Service 151e447614 Internal changes
PiperOrigin-RevId: 496793199
2022-12-20 17:52:17 -08:00
MediaPipe TeamandCopybara-Service 64406a9bf2 Internal change
PiperOrigin-RevId: 496781536
2022-12-20 16:41:49 -08:00
Sebastian SchmidtandCopybara-Service d2f738793c Use uppercase options name for "delegate"
PiperOrigin-RevId: 496764089
2022-12-20 15:18:17 -08:00
MediaPipe TeamandCopybara-Service a7b52d2c52 Internal changes
PiperOrigin-RevId: 496754449
2022-12-20 14:37:18 -08:00
MediaPipe TeamandCopybara-Service ce0bc2b9ac Internal change
PiperOrigin-RevId: 496742964
2022-12-20 14:36:09 -08:00
Sebastian SchmidtandGitHub 55d90e5e9e Merge pull request #3931 from google/assignee-change
Update list of issue assignments
2022-12-20 15:14:07 -07:00
Prianka Liz Kariat ff901a80a5 Added targets in core 2022-12-21 01:24:11 +05:30
Prianka Liz Kariat 4fedea60a9 Added text packet creator 2022-12-21 01:24:02 +05:30
Prianka Liz Kariat e9fc3713f0 Added iOS task runner 2022-12-21 01:23:51 +05:30
Prianka Liz Kariat 64cf5e9b4e Added iOS task options protocol 2022-12-21 01:23:41 +05:30
Prianka Liz Kariat 6d02108bf5 Added task info 2022-12-21 01:23:29 +05:30
Prianka Liz Kariat c56ef735d7 Added classifier options helpers 2022-12-21 01:22:57 +05:30
Prianka Liz Kariat 03bfbca539 Added classifier options 2022-12-21 01:22:44 +05:30
Prianka Liz Kariat e997a19289 Added common utils and string helpers 2022-12-21 01:22:32 +05:30
MediaPipe TeamandCopybara-Service e405c2b67d Internal change
PiperOrigin-RevId: 496702117
2022-12-20 11:04:42 -08:00
MediaPipe TeamandCopybara-Service 8c013647c8 Internal change
PiperOrigin-RevId: 496629682
2022-12-20 04:49:15 -08:00
MediaPipe TeamandCopybara-Service 4682416f0f Internal change
PiperOrigin-RevId: 496568835
2022-12-19 22:10:15 -08:00
Jiuqiang TangandCopybara-Service 9067804005 Fix the missing logging component issue of mediapipe tasks core.
PiperOrigin-RevId: 496548340
2022-12-19 19:41:11 -08:00
MediaPipe TeamandCopybara-Service 994eb28d2c Chain SetName calls where possible
PiperOrigin-RevId: 496534328
2022-12-19 18:07:38 -08:00
MediaPipe TeamandCopybara-Service f5f2fee0b9 Switch to Cast where possible and reduce usage of operator[](port).
PiperOrigin-RevId: 496527250
2022-12-19 17:22:36 -08:00
MediaPipe TeamandCopybara-Service 6842f2c7c6 Use the proper namespace for builder test
PiperOrigin-RevId: 496526588
2022-12-19 17:14:14 -08:00
MediaPipe TeamandCopybara-Service ea0bebc226 Add BGR -> RGB color conversion to ColorConvertCalculator.
PiperOrigin-RevId: 496497002
2022-12-19 14:51:05 -08:00
Nikolay ChirkovandCopybara-Service ef3fa67bf4 Automatic selection of the tensor's storage type by recording previously requested views.
PiperOrigin-RevId: 496466136
2022-12-19 12:37:50 -08:00
MediaPipe TeamandCopybara-Service 3e6cd5d2bf Add support for customizing gesture recognizer layers
PiperOrigin-RevId: 496456160
2022-12-19 11:57:09 -08:00
Sebastian SchmidtandCopybara-Service 4822476974 Internal change
PiperOrigin-RevId: 496443946
2022-12-19 11:07:09 -08:00
Jiuqiang TangandCopybara-Service 7ce4bb72d4 Replace numpy.float with the builtin float type as numpy removes its own float type in v1.24.
PiperOrigin-RevId: 496412858
2022-12-19 09:02:51 -08:00
MediaPipe TeamandCopybara-Service b45554623a Fix typo in GetVectorItemCalculator doc
PiperOrigin-RevId: 495951016
2022-12-16 13:42:50 -08:00
Copybara-Service a864f4e9b3 Merge pull request #3927 from priankakariatyml:ios-options
PiperOrigin-RevId: 495898311
2022-12-16 09:59:10 -08:00
Nikolay ChirkovandCopybara-Service d5562241cc Tensor: Interoperability GPU/Cpu -> Ahwb by transforming the underlying storage into Ahwb with releasing previously Cpu/Gpu resources.
PiperOrigin-RevId: 495748104
2022-12-15 18:34:27 -08:00
MediaPipe TeamandCopybara-Service 0a1f050f1f Internal change
PiperOrigin-RevId: 495741383
2022-12-15 17:52:43 -08:00
Jiuqiang TangandCopybara-Service 6bf5648430 Fix the documentation of the constructor of Image and ImageFrame Python classes.
PiperOrigin-RevId: 495739875
2022-12-15 17:42:23 -08:00
Mark McDonaldandCopybara-Service 8d2473c751 Update Image docs to improve rendering.
The [API docs](https://developers.google.com/mediapipe/api/solutions/python/mp/Image) have a few rendering issues. e.g., the doc generator will turn
```
This block:
  Anything here
```
Into a table with heading `This block` and `Anything here` as a plain-text cell.

In order to render code as code, it needs to be in backticks. They can also be in `>>> code()` format, and we can try to run them ([doctests](https://docs.python.org/3/library/doctest.html)).

I'll have a dashboard ready soon that shows areas we can improve.

PiperOrigin-RevId: 495715576
2022-12-15 15:43:36 -08:00
Khanh LeVietandCopybara-Service 62f0034033 Internal change
PiperOrigin-RevId: 495694817
2022-12-15 14:19:22 -08:00
Jiuqiang TangandCopybara-Service fd50b6aa2f Add a new python unit test to test creating mediapipe Image from cvmat.
PiperOrigin-RevId: 495655719
2022-12-15 11:54:51 -08:00
Jiuqiang TangandCopybara-Service 299aa03302 Internal change
PiperOrigin-RevId: 495613573
2022-12-15 09:24:31 -08:00
MediaPipe TeamandCopybara-Service 6db5eabe0b Internal change
PiperOrigin-RevId: 495525736
2022-12-15 00:44:39 -08:00
Prianka Liz Kariat 5ab17fe686 Removed convenience initializer 2022-12-15 10:28:50 +05:30
Prianka Liz Kariat 163b13d7de Updated comments 2022-12-15 10:23:27 +05:30
Prianka Liz Kariat 9ab0107584 Added new line 2022-12-15 10:21:22 +05:30
Prianka Liz Kariat fe7fbc0b38 Fixed comment 2022-12-15 10:21:14 +05:30
Prianka Liz Kariat bf91c52407 Fixed typos 2022-12-15 10:21:07 +05:30
Nikolay ChirkovandCopybara-Service d526b20e19 Internal change.
PiperOrigin-RevId: 495483878
2022-12-14 19:54:33 -08:00
MediaPipe TeamandCopybara-Service e9e173f9fa Internal change
PiperOrigin-RevId: 495468694
2022-12-14 18:14:55 -08:00
Jiuqiang TangandCopybara-Service 174f2869a3 Internal changes
PiperOrigin-RevId: 495322170
2022-12-14 08:33:45 -08:00
Prianka Liz Kariat c0fed7df31 Added target for task result 2022-12-14 19:15:01 +05:30
Prianka Liz Kariat 0aedff0659 Added target for task options 2022-12-14 19:14:49 +05:30
Prianka Liz Kariat 22bb87d9e0 Added iOS task result 2022-12-14 19:14:11 +05:30
Prianka Liz Kariat e9fb6c28f5 Added task options 2022-12-14 19:14:02 +05:30
Prianka Liz Kariat 7efb3bcf81 Added iOS task error codes 2022-12-14 19:13:41 +05:30
Camillo LugaresiandCopybara-Service db6ea38cf6 Internal change
PiperOrigin-RevId: 495236576
2022-12-14 00:40:01 -08:00
Camillo LugaresiandCopybara-Service 6fa0a58529 Internal change
PiperOrigin-RevId: 495235951
2022-12-14 00:36:16 -08:00
Hadon NashandCopybara-Service b9d020cb7d Internal change
PiperOrigin-RevId: 495163109
2022-12-13 17:00:08 -08:00
MediaPipe TeamandCopybara-Service 904a537b02 Internal change
PiperOrigin-RevId: 495151410
2022-12-13 16:11:10 -08:00
MediaPipe TeamandCopybara-Service d5ff060bfa Internal change
PiperOrigin-RevId: 495149484
2022-12-13 16:04:00 -08:00
Copybara-Service ea0bd77dbf Merge pull request #3911 from priankakariatyml:ios-base-options
PiperOrigin-RevId: 495119958
2022-12-13 14:02:07 -08:00
Hadon NashandCopybara-Service db404b1a85 Internal change
PiperOrigin-RevId: 495058817
2022-12-13 10:23:51 -08:00
Jiuqiang TangandCopybara-Service 78597c5b37 Internal changes.
PiperOrigin-RevId: 495038477
2022-12-13 09:08:15 -08:00
Sebastian SchmidtandCopybara-Service fb21797611 Internal change
PiperOrigin-RevId: 494914168
2022-12-12 21:30:31 -08:00
Mark McDonaldandCopybara-Service 3f66dde8fd Change --site_path default value to match the actual path.
This did not match the URL we ended up using for MediaPipe, so needs to be set correctly in order to generate docs that match the real site.

This change sets the default to be correct.

PiperOrigin-RevId: 494874789
2022-12-12 17:35:16 -08:00
MediaPipe TeamandCopybara-Service 37d2e36960 Internal change
PiperOrigin-RevId: 494791433
2022-12-12 12:10:47 -08:00
Hadon NashandCopybara-Service 421f789ede Internal change
PiperOrigin-RevId: 494420725
2022-12-10 12:33:53 -08:00
Sebastian SchmidtandCopybara-Service e9bb51a524 Internal change
PiperOrigin-RevId: 494314595
2022-12-09 19:22:06 -08:00
Khanh LeVietandCopybara-Service edafef9fd8 Updated issue templates.
PiperOrigin-RevId: 494305235
2022-12-09 18:14:45 -08:00
Sebastian SchmidtandCopybara-Service 69c3c4c181 Internal change
PiperOrigin-RevId: 494305195
2022-12-09 18:10:45 -08:00
Jiuqiang TangandCopybara-Service 453d67de92 Add MergeDetectionsToVectorCalculator.
PiperOrigin-RevId: 494246359
2022-12-09 13:12:57 -08:00
Nikolay ChirkovandCopybara-Service db3cb68d91 Internal change.
PiperOrigin-RevId: 494166776
2022-12-09 07:29:01 -08:00
MediaPipe TeamandCopybara-Service 5bc1baf96a Internal change
PiperOrigin-RevId: 494150467
2022-12-09 05:57:12 -08:00
MediaPipe TeamandCopybara-Service 4c4df2cf18 Internal change for profiling
PiperOrigin-RevId: 494135244
2022-12-09 04:13:02 -08:00
MediaPipe TeamandCopybara-Service 3aeec84ac0 Internal change for profiling
PiperOrigin-RevId: 494126771
2022-12-09 03:21:10 -08:00
Nikolay ChirkovandCopybara-Service bea0caae65 Tensor: Cpu -> Ahwb storage transfer
PiperOrigin-RevId: 494033280
2022-12-08 17:06:56 -08:00
Sebastian SchmidtandCopybara-Service 05535db5f7 Fix assertion failure in Hair Segmentation demo
PiperOrigin-RevId: 494004801
2022-12-08 15:03:39 -08:00
MediaPipe TeamandCopybara-Service b4e1969e43 Add pip package builder for model_maker
PiperOrigin-RevId: 493989013
2022-12-08 14:03:15 -08:00
Nikolay ChirkovandCopybara-Service 0fbaa8dc8a Internal change.
PiperOrigin-RevId: 493973435
2022-12-08 13:01:52 -08:00
MediaPipe TeamandCopybara-Service a641ea12e1 Update gesture recognizer to new mediapipe tasks pipeline
PiperOrigin-RevId: 493950564
2022-12-08 11:32:42 -08:00
Jiuqiang TangandCopybara-Service 13f8fa5139 Retire the visibility group "//mediapipe/framework:mediapipe_internal" in the "mediapipe/calculators/tensor" dir.
PiperOrigin-RevId: 493895834
2022-12-08 08:03:47 -08:00
Prianka Liz Kariat 66dbd9969a Updated license text 2022-12-08 10:25:01 +05:30
Prianka Liz Kariat d1820320b1 Added base options 2022-12-08 10:23:53 +05:30
Sebastian SchmidtandCopybara-Service 9ae2e43b70 Open Source the remaining MediaPipe Tasks tests for Web
PiperOrigin-RevId: 493769657
2022-12-07 19:19:27 -08:00
Adam CozzetteandCopybara-Service 24c8fa97e9 Internal change
PiperOrigin-RevId: 493768013
2022-12-07 19:06:00 -08:00
MediaPipe TeamandCopybara-Service 700c7b4b22 Internal refactoring for TextEmbedder.
PiperOrigin-RevId: 493766612
2022-12-07 18:56:27 -08:00
MediaPipe TeamandCopybara-Service a0efcb47f2 internal change
PiperOrigin-RevId: 493742399
2022-12-07 16:38:42 -08:00
MediaPipe TeamandCopybara-Service a59f0a9924 Make java/C++/python tasks API public visible.
PiperOrigin-RevId: 493730506
2022-12-07 15:51:04 -08:00
Sebastian SchmidtandCopybara-Service 5f97b29b3b Update Bazel dependencies for Apple
PiperOrigin-RevId: 493723833
2022-12-07 15:25:56 -08:00
MediaPipe TeamandCopybara-Service 91664eb254 Object Detector deduplication
PiperOrigin-RevId: 493716159
2022-12-07 14:54:30 -08:00
Jiuqiang TangandCopybara-Service ef1507ed5d Retire the visibility group "//mediapipe/framework:mediapipe_internal".
PiperOrigin-RevId: 493713823
2022-12-07 14:45:49 -08:00
Nikolay ChirkovandCopybara-Service 7faee517c4 Tensor: Move general CPU/SSBO tensor storage into Ahwb-backed CPU/SSBO storage.
PiperOrigin-RevId: 493710495
2022-12-07 14:32:42 -08:00
Nikolay ChirkovandCopybara-Service ea74db86dd Tensor: clang tidy fixes.
PiperOrigin-RevId: 493703073
2022-12-07 14:06:29 -08:00
Jiuqiang TangandCopybara-Service 955f090f9f Retire the visibility group "//mediapipe/framework:mediapipe_internal".
PiperOrigin-RevId: 493687025
2022-12-07 13:08:33 -08:00
Sebastian SchmidtandCopybara-Service 2811e0c5c8 Open Source the first set of MediaPipe Tasks tests for Web
PiperOrigin-RevId: 493673279
2022-12-07 12:15:34 -08:00
Jiuqiang TangandCopybara-Service 3c0ddf16b4 Fix an incorrect model sanity check in the object detector graph.
PiperOrigin-RevId: 493663592
2022-12-07 11:38:54 -08:00
Jiuqiang TangandCopybara-Service 80c605459c Open up framework visibility.
PiperOrigin-RevId: 493660013
2022-12-07 11:26:16 -08:00
Sebastian SchmidtandCopybara-Service d84eec387b Add missing import to InferenceCalculator.proto
PiperOrigin-RevId: 493649869
2022-12-07 10:52:17 -08:00
Khanh LeVietandCopybara-Service d9688b769f Hide internal APIs from mediapipe pip package's API docs.
PiperOrigin-RevId: 493607984
2022-12-07 08:15:25 -08:00
Nikolay ChirkovandCopybara-Service aad797197b TensorV1 EGL.h include fix.
PiperOrigin-RevId: 493596083
2022-12-07 07:16:59 -08:00
866 changed files with 70077 additions and 4225 deletions
@@ -1,27 +0,0 @@
---
name: "Build/Installation Issue"
about: Use this template for build/installation issues
labels: type:build/install
---
<em>Please make sure that this is a build/installation issue and also refer to the [troubleshooting](https://google.github.io/mediapipe/getting_started/troubleshooting.html) documentation before raising any issues.</em>
**System information** (Please provide as much relevant information as possible)
- OS Platform and Distribution (e.g. Linux Ubuntu 16.04, Android 11, iOS 14.4):
- Compiler version (e.g. gcc/g++ 8 /Apple clang version 12.0.0):
- Programming Language and version ( e.g. C++ 14, Python 3.6, Java ):
- Installed using virtualenv? pip? Conda? (if python):
- [MediaPipe version](https://github.com/google/mediapipe/releases):
- Bazel version:
- XCode and Tulsi versions (if iOS):
- Android SDK and NDK versions (if android):
- Android [AAR](https://google.github.io/mediapipe/getting_started/android_archive_library.html) ( if android):
- OpenCV version (if running on desktop):
**Describe the problem**:
**[Provide the exact sequence of commands / steps that you executed before running into the problem](https://google.github.io/mediapipe/getting_started/getting_started.html):**
**Complete Logs:**
Include Complete Log information or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached:
@@ -1,26 +0,0 @@
---
name: "Solution Issue"
about: Use this template for assistance with a specific mediapipe solution, such as "Pose" or "Iris", including inference model usage/training, solution-specific calculators, etc.
labels: type:support
---
<em>Please make sure that this is a [solution](https://google.github.io/mediapipe/solutions/solutions.html) issue.<em>
**System information** (Please provide as much relevant information as possible)
- Have I written custom code (as opposed to using a stock example script provided in Mediapipe):
- OS Platform and Distribution (e.g., Linux Ubuntu 16.04, Android 11, iOS 14.4):
- [MediaPipe version](https://github.com/google/mediapipe/releases):
- Bazel version:
- Solution (e.g. FaceMesh, Pose, Holistic):
- Programming Language and version ( e.g. C++, Python, Java):
**Describe the expected behavior:**
**Standalone code you may have used to try to get what you need :**
If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab/repo link /any notebook:
**Other info / Complete Logs :**
Include any logs or source code that would be helpful to
diagnose the problem. If including tracebacks, please include the full
traceback. Large logs and files should be attached:
-25
View File
@@ -1,25 +0,0 @@
---
name: "Tasks Issue"
about: Use this template for assistance with using MediaPipe Tasks to deploy on-device ML solutions (e.g. gesture recognition etc.) on supported platforms.
labels: type:support
---
<em>Please make sure that this is a [Tasks](https://developers.google.com/mediapipe/solutions) issue.<em>
**System information** (Please provide as much relevant information as possible)
- Have I written custom code (as opposed to using a stock example script provided in MediaPipe):
- OS Platform and Distribution (e.g., Linux Ubuntu 16.04, Android 11, iOS 14.4):
- MediaPipe Tasks SDK version:
- Task name (e.g. Object detection, Gesture recognition etc.):
- Programming Language and version ( e.g. C++, Python, Java):
**Describe the expected behavior:**
**Standalone code you may have used to try to get what you need :**
If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem:
**Other info / Complete Logs :**
Include any logs or source code that would be helpful to
diagnose the problem. If including tracebacks, please include the full
traceback. Large logs and files should be attached:
@@ -1,25 +0,0 @@
---
name: "Model Maker Issue"
about: Use this template for assistance with using MediaPipe Model Maker to create custom on-device ML solutions.
labels: type:support
---
<em>Please make sure that this is a [Model Maker](https://developers.google.com/mediapipe/solutions) issue.<em>
**System information** (Please provide as much relevant information as possible)
- Have I written custom code (as opposed to using a stock example script provided in MediaPipe):
- OS Platform and Distribution (e.g., Linux Ubuntu 16.04):
- Python version (e.g. 3.8):
- [MediaPipe Model Maker version](https://pypi.org/project/mediapipe-model-maker/):
- Task name (e.g. Image classification, Gesture recognition etc.):
**Describe the expected behavior:**
**Standalone code you may have used to try to get what you need :**
If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem:
**Other info / Complete Logs :**
Include any logs or source code that would be helpful to
diagnose the problem. If including tracebacks, please include the full
traceback. Large logs and files should be attached:
@@ -1,51 +0,0 @@
---
name: "Documentation Issue"
about: Use this template for documentation related issues
labels: type:docs
---
Thank you for submitting a MediaPipe documentation issue.
The MediaPipe docs are open source! To get involved, read the documentation Contributor Guide
## URL(s) with the issue:
Please provide a link to the documentation entry, for example: https://github.com/google/mediapipe/blob/master/docs/solutions/face_mesh.md#models
## Description of issue (what needs changing):
Kinds of documentation problems:
### Clear description
For example, why should someone use this method? How is it useful?
### Correct links
Is the link to the source code correct?
### Parameters defined
Are all parameters defined and formatted correctly?
### Returns defined
Are return values defined?
### Raises listed and defined
Are the errors defined? For example,
### Usage example
Is there a usage example?
See the API guide:
on how to write testable usage examples.
### Request visuals, if applicable
Are there currently visuals? If not, will it clarify the content?
### Submit a pull request?
Are you planning to also submit a pull request to fix the issue? See the docs
https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md
-32
View File
@@ -1,32 +0,0 @@
---
name: "Bug Issue"
about: Use this template for reporting a bug
labels: type:bug
---
<em>Please make sure that this is a bug and also refer to the [troubleshooting](https://google.github.io/mediapipe/getting_started/troubleshooting.html), FAQ documentation before raising any issues.</em>
**System information** (Please provide as much relevant information as possible)
- Have I written custom code (as opposed to using a stock example script provided in MediaPipe):
- OS Platform and Distribution (e.g., Linux Ubuntu 16.04, Android 11, iOS 14.4):
- Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device:
- Browser and version (e.g. Google Chrome, Safari) if the issue happens on browser:
- Programming Language and version ( e.g. C++, Python, Java):
- [MediaPipe version](https://github.com/google/mediapipe/releases):
- Bazel version (if compiling from source):
- Solution ( e.g. FaceMesh, Pose, Holistic ):
- Android Studio, NDK, SDK versions (if issue is related to building in Android environment):
- Xcode & Tulsi version (if issue is related to building for iOS):
**Describe the current behavior:**
**Describe the expected behavior:**
**Standalone code to reproduce the issue:**
Provide a reproducible test case that is the bare minimum necessary to replicate the problem. If possible, please share a link to Colab/repo link /any notebook:
**Other info / Complete Logs :**
Include any logs or source code that would be helpful to
diagnose the problem. If including tracebacks, please include the full
traceback. Large logs and files should be attached
@@ -1,24 +0,0 @@
---
name: "Feature Request"
about: Use this template for raising a feature request
labels: type:feature
---
<em>Please make sure that this is a feature request.</em>
**System information** (Please provide as much relevant information as possible)
- MediaPipe Solution (you are using):
- Programming language : C++/typescript/Python/Objective C/Android Java
- Are you willing to contribute it (Yes/No):
**Describe the feature and the current behavior/state:**
**Will this change the current api? How?**
**Who will benefit with this feature?**
**Please specify the use cases for this feature:**
**Any Other info:**
@@ -0,0 +1,73 @@
name: Documentation issue
description: Use this template for documentation related issues. If this doesnt look right, choose a different type.
labels: 'type:doc-bug'
body:
- type: markdown
id: link
attributes:
value: Thank you for submitting a MediaPipe documentation issue. The MediaPipe docs are open source! To get involved, read the documentation Contributor Guide
- type: markdown
id: url
attributes:
value: URL(s) with the issue Please provide a link to the documentation entry, for example https://github.com/google/mediapipe/blob/master/docs/solutions/face_mesh.md#models
- type: input
id: description
attributes:
label: Description of issue (what needs changing)
description: Kinds of documentation problems
- type: input
id: clear_desc
attributes:
label: Clear description
description: For example, why should someone use this method? How is it useful?
validations:
required: true
- type: input
id: link
attributes:
label: Correct links
description: Is the link to the source code correct?
validations:
required: false
- type: input
id: parameter
attributes:
label: Parameters defined
description: Are all parameters defined and formatted correctly?
validations:
required: false
- type: input
id: returns
attributes:
label: Returns defined
description: Are return values defined?
validations:
required: false
- type: input
id: raises
attributes:
label: Raises listed and defined
description: Are the errors defined? For example,
validations:
required: false
- type: input
id: usage
attributes:
label: Usage example
description: Is there a usage example? See the API guide-on how to write testable usage examples.
validations:
required: false
- type: input
id: visual
attributes:
label: Request visuals, if applicable
description: Are there currently visuals? If not, will it clarify the content?
validations:
required: false
- type: input
id: pull
attributes:
label: Submit a pull request?
description: Are you planning to also submit a pull request to fix the issue? See the [docs](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md)
validations:
required: false
@@ -0,0 +1,80 @@
name: Solution(Legacy) Issue
description: Use this template for assistance with a specific Mediapipe solution (google.github.io/mediapipe/solutions) such as "Pose", including inference model usage/training, solution-specific calculators etc.
labels: 'type:support'
body:
- type: markdown
id: linkmodel
attributes:
value: Please make sure that this is a [solution](https://google.github.io/mediapipe/solutions/solutions.html) issue.
- type: dropdown
id: customcode_model
attributes:
label: Have I written custom code (as opposed to using a stock example script provided in MediaPipe)
options:
- 'Yes'
- 'No'
validations:
required: false
- type: input
id: os_model
attributes:
label: OS Platform and Distribution
placeholder: e.g. Linux Ubuntu 16.04, Android 11, iOS 14.4
validations:
required: false
- type: input
id: mediapipe_version
attributes:
label: MediaPipe version
validations:
required: false
- type: input
id: bazel_version
attributes:
label: Bazel version
validations:
required: false
- type: input
id: solution
attributes:
label: Solution
placeholder: e.g. FaceMesh, Pose, Holistic
validations:
required: false
- type: input
id: programminglang
attributes:
label: Programming Language and version
placeholder: e.g. C++, Python, Java
validations:
required: false
- type: textarea
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: false
- type: textarea
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: false
- type: textarea
id: what-happened_model
attributes:
label: Standalone code/steps you may have used to try to get what you need
description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem
render: shell
validations:
required: false
- type: textarea
id: other_info
attributes:
label: Other info / Complete Logs
description: Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached
render: shell
validations:
required: false
@@ -0,0 +1,112 @@
name: Bug Issues
description: Use this template for reporting a bug. If this doesnt look right, choose a different type.
labels: 'type:bug'
body:
- type: markdown
id: link
attributes:
value: Please make sure that this is a bug and also refer to the [troubleshooting](https://google.github.io/mediapipe/getting_started/troubleshooting.html), FAQ documentation before raising any issues.
- type: dropdown
id: customcode_model
attributes:
label: Have I written custom code (as opposed to using a stock example script provided in MediaPipe)
options:
- 'Yes'
- 'No'
validations:
required: false
- type: input
id: os
attributes:
label: OS Platform and Distribution
description:
placeholder: e.g. Linux Ubuntu 16.04, Android 11, iOS 14.4
validations:
required: true
- type: input
id: mobile_device
attributes:
label: Mobile device if the issue happens on mobile device
description:
placeholder: e.g. iPhone 8, Pixel 2, Samsung Galaxy
validations:
required: false
- type: input
id: browser_version
attributes:
label: Browser and version if the issue happens on browser
placeholder: e.g. Google Chrome 109.0.5414.119, Safari 16.3
validations:
required: false
- type: input
id: programminglang
attributes:
label: Programming Language and version
placeholder: e.g. C++, Python, Java
validations:
required: true
- type: input
id: mediapipever
attributes:
label: MediaPipe version
description:
placeholder: e.g. 0.8.11, 0.9.1
validations:
required: false
- type: input
id: bazelver
attributes:
label: Bazel version
description:
placeholder: e.g. 5.0, 5.1
validations:
required: false
- type: input
id: solution
attributes:
label: Solution
placeholder: e.g. FaceMesh, Pose, Holistic
validations:
required: true
- type: input
id: sdkndkversion
attributes:
label: Android Studio, NDK, SDK versions (if issue is related to building in Android environment)
validations:
required: false
- type: input
id: xcode_ver
attributes:
label: Xcode & Tulsi version (if issue is related to building for iOS)
validations:
required: false
- type: textarea
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: true
- type: textarea
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: true
- type: textarea
id: what-happened_model
attributes:
label: Standalone code/steps you may have used to try to get what you need
description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem
render: shell
validations:
required: true
- type: textarea
id: other_info
attributes:
label: Other info / Complete Logs
description: Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached
render: shell
validations:
required: false
@@ -0,0 +1,109 @@
name: Build/Install Issue
description: Use this template to report build/install issue
labels: 'type:build/install'
body:
- type: markdown
id: link
attributes:
value: Please make sure that this is a build/installation issue and also refer to the [troubleshooting](https://google.github.io/mediapipe/getting_started/troubleshooting.html) documentation before raising any issues.
- type: input
id: os
attributes:
label: OS Platform and Distribution
description:
placeholder: e.g. Linux Ubuntu 16.04, Android 11, iOS 14.4
validations:
required: true
- type: input
id: compilerversion
attributes:
label: Compiler version
description:
placeholder: e.g. gcc/g++ 8 /Apple clang version 12.0.0
validations:
required: false
- type: input
id: programminglang
attributes:
label: Programming Language and version
description:
placeholder: e.g. C++ 14, Python 3.6, Java
validations:
required: true
- type: input
id: virtualenv
attributes:
label: Installed using virtualenv? pip? Conda?(if python)
description:
placeholder:
validations:
required: false
- type: input
id: mediapipever
attributes:
label: MediaPipe version
description:
placeholder: e.g. 0.8.11, 0.9.1
validations:
required: false
- type: input
id: bazelver
attributes:
label: Bazel version
description:
placeholder: e.g. 5.0, 5.1
validations:
required: false
- type: input
id: xcodeversion
attributes:
label: XCode and Tulsi versions (if iOS)
description:
placeholder:
validations:
required: false
- type: input
id: sdkndkversion
attributes:
label: Android SDK and NDK versions (if android)
description:
placeholder:
validations:
required: false
- type: dropdown
id: androidaar
attributes:
label: Android AAR (if android)
options:
- 'Yes'
- 'No'
validations:
required: false
- type: input
id: opencvversion
attributes:
label: OpenCV version (if running on desktop)
description:
placeholder:
validations:
required: false
- type: textarea
id: what-happened
attributes:
label: Describe the problem
description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html)
placeholder: Tell us what you see!
value: "A bug happened!"
render: shell
validations:
required: true
- type: textarea
id: code-to-reproduce
attributes:
label: Complete Logs
description: Include Complete Log information or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached
placeholder: Tell us what you see!
value:
render: shell
validations:
required: true
@@ -0,0 +1,64 @@
name: Feature Request Issues
description: Use this template for raising a feature request. If this doesnt look right, choose a different type.
labels: 'type:feature'
body:
- type: markdown
id: linkmodel
attributes:
value: Please make sure that this is a feature request.
- type: input
id: solution
attributes:
label: MediaPipe Solution (you are using)
validations:
required: false
- type: input
id: pgmlang
attributes:
label: Programming language
placeholder: C++/typescript/Python/Objective C/Android Java
validations:
required: false
- type: dropdown
id: willingcon
attributes:
label: Are you willing to contribute it
options:
- 'Yes'
- 'No'
validations:
required: false
- type: textarea
id: behaviour
attributes:
label: Describe the feature and the current behaviour/state
render: shell
validations:
required: true
- type: textarea
id: api_change
attributes:
label: Will this change the current API? How?
render: shell
validations:
required: false
- type: textarea
id: benifit
attributes:
label: Who will benefit with this feature?
validations:
required: false
- type: textarea
id: use_case
attributes:
label: Please specify the use cases for this feature
render: shell
validations:
required: true
- type: textarea
id: info_other
attributes:
label: Any Other info
render: shell
validations:
required: false
@@ -0,0 +1,73 @@
name: Model Maker Issues
description: Use this template for assistance with using MediaPipe Model Maker (developers.google.com/mediapipe/solutions) to create custom on-device ML solutions.
labels: 'type:modelmaker'
body:
- type: markdown
id: linkmodel
attributes:
value: Please make sure that this is a [Model Maker](https://developers.google.com/mediapipe/solutions) issue
- type: dropdown
id: customcode_model
attributes:
label: Have I written custom code (as opposed to using a stock example script provided in MediaPipe)
options:
- 'Yes'
- 'No'
validations:
required: false
- type: input
id: os_model
attributes:
label: OS Platform and Distribution
placeholder: e.g. Linux Ubuntu 16.04, Android 11, iOS 14.4
validations:
required: true
- type: input
id: pythonver
attributes:
label: Python Version
placeholder: e.g. 3.7, 3.8
validations:
required: true
- type: input
id: modelmakerver
attributes:
label: MediaPipe Model Maker version
validations:
required: false
- type: input
id: taskname
attributes:
label: Task name (e.g. Image classification, Gesture recognition etc.)
validations:
required: true
- type: textarea
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: true
- type: textarea
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: true
- type: textarea
id: what-happened_model
attributes:
label: Standalone code/steps you may have used to try to get what you need
description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem
render: shell
validations:
required: true
- type: textarea
id: other_info
attributes:
label: Other info / Complete Logs
description: Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached
render: shell
validations:
required: false
@@ -0,0 +1,63 @@
name: Studio Issues
description: Use this template for assistance with the MediaPipe Studio application. If this doesnt look right, choose a different type.
labels: 'type:support'
body:
- type: markdown
id: linkmodel
attributes:
value: Please make sure that this is a MediaPipe Studio issue.
- type: input
id: os_model
attributes:
label: OS Platform and Distribution
placeholder: e.g. Linux Ubuntu 16.04, Android 11, iOS 14.4
validations:
required: false
- type: input
id: browserver
attributes:
label: Browser and Version
validations:
required: false
- type: input
id: hardware
attributes:
label: Any microphone or camera hardware
validations:
required: false
- type: input
id: url
attributes:
label: URL that shows the problem
validations:
required: false
- type: textarea
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: false
- type: textarea
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: false
- type: textarea
id: what-happened_model
attributes:
label: Standalone code/steps you may have used to try to get what you need
description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem
render: shell
validations:
required: false
- type: textarea
id: other_info
attributes:
label: Other info / Complete Logs
description: Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached
render: shell
validations:
required: false
@@ -0,0 +1,72 @@
name: Task Issue
description: Use this template for assistance with using MediaPipe Tasks (developers.google.com/mediapipe/solutions) to deploy on-device ML solutions (e.g. gesture recognition etc.) on supported platforms
labels: 'type:task'
body:
- type: markdown
id: linkmodel
attributes:
value: Please make sure that this is a [Tasks](https://developers.google.com/mediapipe/solutions) issue.
- type: dropdown
id: customcode_model
attributes:
label: Have I written custom code (as opposed to using a stock example script provided in MediaPipe)
options:
- 'Yes'
- 'No'
validations:
required: false
- type: input
id: os_model
attributes:
label: OS Platform and Distribution
placeholder: e.g. Linux Ubuntu 16.04, Android 11, iOS 14.4
validations:
required: true
- type: input
id: task-sdk-version
attributes:
label: MediaPipe Tasks SDK version
validations:
required: false
- type: input
id: taskname
attributes:
label: Task name (e.g. Image classification, Gesture recognition etc.)
validations:
required: true
- type: input
id: programminglang
attributes:
label: Programming Language and version (e.g. C++, Python, Java)
validations:
required: true
- type: textarea
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: true
- type: textarea
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: true
- type: textarea
id: what-happened_model
attributes:
label: Standalone code/steps you may have used to try to get what you need
description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem
render: shell
validations:
required: true
- type: textarea
id: other_info
attributes:
label: Other info / Complete Logs
description: Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached
render: shell
validations:
required: false
+11
View File
@@ -19,6 +19,17 @@ ML solutions for live and streaming media.
![ready_to_use.png](https://mediapipe.dev/images/ready_to_use_small.png) | ![open_source.png](https://mediapipe.dev/images/open_source_small.png) ![ready_to_use.png](https://mediapipe.dev/images/ready_to_use_small.png) | ![open_source.png](https://mediapipe.dev/images/open_source_small.png)
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable* ***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
----
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023.*
*This notice and web page will be removed on April 3, 2023.*
----
## ML solutions in MediaPipe ## ML solutions in MediaPipe
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
+87 -54
View File
@@ -10,33 +10,31 @@ bind(
http_archive( http_archive(
name = "bazel_skylib", name = "bazel_skylib",
type = "tar.gz", sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506",
urls = [ urls = [
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
], ],
sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
) )
load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
bazel_skylib_workspace() bazel_skylib_workspace()
load("@bazel_skylib//lib:versions.bzl", "versions") load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "3.7.2") versions.check(minimum_bazel_version = "3.7.2")
# ABSL cpp library lts_2021_03_24, patch 2. # ABSL cpp library lts_2023_01_25.
http_archive( http_archive(
name = "com_google_absl", name = "com_google_absl",
urls = [ urls = [
"https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz", "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.0.tar.gz",
], ],
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
patches = [ patches = [
"@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff" "@//third_party:com_google_absl_windows_patch.diff"
], ],
patch_args = [ patch_args = [
"-p1", "-p1",
], ],
strip_prefix = "abseil-cpp-20220623.1", strip_prefix = "abseil-cpp-20230125.0",
sha256 = "91ac87d30cc6d79f9ab974c51874a704de9c2647c40f6932597329a282217ba8" sha256 = "3ea49a7d97421b88a8c48a0de16c16048e17725c7ec0f1d3ea2683a2a75adc21"
) )
http_archive( http_archive(
@@ -148,13 +146,14 @@ http_archive(
], ],
) )
load("//third_party/flatbuffers:workspace.bzl", flatbuffers = "repo") load("@//third_party/flatbuffers:workspace.bzl", flatbuffers = "repo")
flatbuffers() flatbuffers()
http_archive( http_archive(
name = "com_google_audio_tools", name = "com_google_audio_tools",
strip_prefix = "multichannel-audio-tools-master", strip_prefix = "multichannel-audio-tools-1f6b1319f13282eda6ff1317be13de67f4723860",
urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"], urls = ["https://github.com/google/multichannel-audio-tools/archive/1f6b1319f13282eda6ff1317be13de67f4723860.zip"],
sha256 = "fe346e1aee4f5069c4cbccb88706a9a2b2b4cf98aeb91ec1319be77e07dd7435",
repo_mapping = {"@com_github_glog_glog" : "@com_github_glog_glog_no_gflags"}, repo_mapping = {"@com_github_glog_glog" : "@com_github_glog_glog_no_gflags"},
) )
@@ -174,7 +173,7 @@ http_archive(
"https://github.com/google/sentencepiece/archive/1.0.0.zip", "https://github.com/google/sentencepiece/archive/1.0.0.zip",
], ],
patches = [ patches = [
"//third_party:com_google_sentencepiece_no_gflag_no_gtest.diff", "@//third_party:com_google_sentencepiece_no_gflag_no_gtest.diff",
], ],
patch_args = ["-p1"], patch_args = ["-p1"],
repo_mapping = {"@com_google_glog" : "@com_github_glog_glog_no_gflags"}, repo_mapping = {"@com_google_glog" : "@com_github_glog_glog_no_gflags"},
@@ -188,8 +187,8 @@ http_archive(
"https://github.com/tensorflow/text/archive/v2.2.0.zip", "https://github.com/tensorflow/text/archive/v2.2.0.zip",
], ],
patches = [ patches = [
"//third_party:tensorflow_text_remove_tf_deps.diff", "@//third_party:tensorflow_text_remove_tf_deps.diff",
"//third_party:tensorflow_text_a0f49e63.diff", "@//third_party:tensorflow_text_a0f49e63.diff",
], ],
patch_args = ["-p1"], patch_args = ["-p1"],
repo_mapping = {"@com_google_re2": "@com_googlesource_code_re2"}, repo_mapping = {"@com_google_re2": "@com_googlesource_code_re2"},
@@ -320,12 +319,30 @@ http_archive(
], ],
) )
# iOS basic build deps. # Load Zlib before initializing TensorFlow and the iOS build rules to guarantee
# that the target @zlib//:mini_zlib is available
http_archive(
name = "zlib",
build_file = "@//third_party:zlib.BUILD",
sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
strip_prefix = "zlib-1.2.11",
urls = [
"http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
"http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
],
patches = [
"@//third_party:zlib.diff",
],
patch_args = [
"-p1",
],
)
# iOS basic build deps.
http_archive( http_archive(
name = "build_bazel_rules_apple", name = "build_bazel_rules_apple",
sha256 = "77e8bf6fda706f420a55874ae6ee4df0c9d95da6c7838228b26910fc82eea5a2", sha256 = "f94e6dddf74739ef5cb30f000e13a2a613f6ebfa5e63588305a71fce8a8a9911",
url = "https://github.com/bazelbuild/rules_apple/releases/download/0.32.0/rules_apple.0.32.0.tar.gz", url = "https://github.com/bazelbuild/rules_apple/releases/download/1.1.3/rules_apple.1.1.3.tar.gz",
patches = [ patches = [
# Bypass checking ios unit test runner when building MP ios applications. # Bypass checking ios unit test runner when building MP ios applications.
"@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff" "@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
@@ -339,29 +356,24 @@ load(
"@build_bazel_rules_apple//apple:repositories.bzl", "@build_bazel_rules_apple//apple:repositories.bzl",
"apple_rules_dependencies", "apple_rules_dependencies",
) )
apple_rules_dependencies() apple_rules_dependencies()
load( load(
"@build_bazel_rules_swift//swift:repositories.bzl", "@build_bazel_rules_swift//swift:repositories.bzl",
"swift_rules_dependencies", "swift_rules_dependencies",
) )
swift_rules_dependencies() swift_rules_dependencies()
http_archive( load(
name = "build_bazel_apple_support", "@build_bazel_rules_swift//swift:extras.bzl",
sha256 = "741366f79d900c11e11d8efd6cc6c66a31bfb2451178b58e0b5edc6f1db17b35", "swift_rules_extra_dependencies",
urls = [
"https://github.com/bazelbuild/apple_support/releases/download/0.10.0/apple_support.0.10.0.tar.gz"
],
) )
swift_rules_extra_dependencies()
load( load(
"@build_bazel_apple_support//lib:repositories.bzl", "@build_bazel_apple_support//lib:repositories.bzl",
"apple_support_dependencies", "apple_support_dependencies",
) )
apple_support_dependencies() apple_support_dependencies()
# More iOS deps. # More iOS deps.
@@ -442,29 +454,10 @@ http_archive(
], ],
) )
# Load Zlib before initializing TensorFlow to guarantee that the target
# @zlib//:mini_zlib is available
http_archive(
name = "zlib",
build_file = "//third_party:zlib.BUILD",
sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
strip_prefix = "zlib-1.2.11",
urls = [
"http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
"http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
],
patches = [
"@//third_party:zlib.diff",
],
patch_args = [
"-p1",
],
)
# TensorFlow repo should always go after the other external dependencies. # TensorFlow repo should always go after the other external dependencies.
# TF on 2022-08-10. # TF on 2023-03-08.
_TENSORFLOW_GIT_COMMIT = "af1d5bc4fbb66d9e6cc1cf89503014a99233583b" _TENSORFLOW_GIT_COMMIT = "24f7ee636d62e1f8d8330357f8bbd65956dfb84d"
_TENSORFLOW_SHA256 = "f85a5443264fc58a12d136ca6a30774b5bc25ceaf7d114d97f252351b3c3a2cb" _TENSORFLOW_SHA256 = "7f8a96dd99215c0cdc77230d3dbce43e60102b64a89203ad04aa09b0a187a4bd"
http_archive( http_archive(
name = "org_tensorflow", name = "org_tensorflow",
urls = [ urls = [
@@ -518,8 +511,8 @@ load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install
node_repositories() node_repositories()
yarn_install( yarn_install(
name = "npm", name = "npm",
package_json = "//:package.json", package_json = "@//:package.json",
yarn_lock = "//:yarn.lock", yarn_lock = "@//:yarn.lock",
) )
# Protobuf for Node dependencies # Protobuf for Node dependencies
@@ -545,8 +538,48 @@ load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_
rules_proto_dependencies() rules_proto_dependencies()
rules_proto_toolchains() rules_proto_toolchains()
load("//third_party:external_files.bzl", "external_files") load("@//third_party:external_files.bzl", "external_files")
external_files() external_files()
load("//third_party:wasm_files.bzl", "wasm_files") load("@//third_party:wasm_files.bzl", "wasm_files")
wasm_files() wasm_files()
# Halide
new_local_repository(
name = "halide",
build_file = "@//third_party/halide:BUILD.bazel",
path = "third_party/halide"
)
http_archive(
name = "linux_halide",
sha256 = "f62b2914823d6e33d18693f5b74484f274523bf5402ce51988e24393d123b375",
strip_prefix = "Halide-15.0.0-x86-64-linux",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.0/Halide-15.0.0-x86-64-linux-d7651f4b32f9dbd764f243134001f7554378d62d.tar.gz"],
build_file = "@//third_party:halide.BUILD",
)
http_archive(
name = "macos_x86_64_halide",
sha256 = "3d832aed942080ea89aa832462c68fbb906f3055c440b7b6d35093d7c52f6aab",
strip_prefix = "Halide-15.0.0-x86-64-osx",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.0/Halide-15.0.0-x86-64-osx-d7651f4b32f9dbd764f243134001f7554378d62d.tar.gz"],
build_file = "@//third_party:halide.BUILD",
)
http_archive(
name = "macos_arm_64_halide",
sha256 = "b1fad3c9810122b187303d7031d9e35fb43761f345d18cc4492c00ed5877f641",
strip_prefix = "Halide-15.0.0-arm-64-osx",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.0/Halide-15.0.0-arm-64-osx-d7651f4b32f9dbd764f243134001f7554378d62d.tar.gz"],
build_file = "@//third_party:halide.BUILD",
)
http_archive(
name = "windows_halide",
sha256 = "5acf6fe161dd375856a2b43f4bb0a32815ba958b0585ee312c44e008aa7b0b64",
strip_prefix = "Halide-15.0.0-x86-64-windows",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.0/Halide-15.0.0-x86-64-windows-d7651f4b32f9dbd764f243134001f7554378d62d.zip"],
build_file = "@//third_party:halide.BUILD",
)
+1 -3
View File
@@ -4,12 +4,10 @@ py_binary(
name = "build_py_api_docs", name = "build_py_api_docs",
srcs = ["build_py_api_docs.py"], srcs = ["build_py_api_docs.py"],
deps = [ deps = [
"//mediapipe",
"//third_party/py/absl:app", "//third_party/py/absl:app",
"//third_party/py/absl/flags", "//third_party/py/absl/flags",
"//third_party/py/tensorflow_docs", "//third_party/py/mediapipe",
"//third_party/py/tensorflow_docs/api_generator:generate_lib", "//third_party/py/tensorflow_docs/api_generator:generate_lib",
"//third_party/py/tensorflow_docs/api_generator:public_api",
], ],
) )
+13
View File
@@ -0,0 +1,13 @@
<html lang="en">
<head>
<meta charset="utf-8"/>
<meta http-equiv="refresh" content="0;url={{ page.target }}"/>
<link rel="canonical" href="{{ page.target }}"/>
<title>Redirecting</title>
</head>
<body>
<p>This page now lives on https://developers.google.com/mediapipe/. If you aren't automatically
redirected, follow this
<a href="{{ page.target }}">link</a>.</p>
</body>
</html>
+2 -2
View File
@@ -44,14 +44,14 @@ _OUTPUT_DIR = flags.DEFINE_string(
_URL_PREFIX = flags.DEFINE_string( _URL_PREFIX = flags.DEFINE_string(
'code_url_prefix', 'code_url_prefix',
'https://github.com/google/mediapipe/tree/master/mediapipe', 'https://github.com/google/mediapipe/blob/master/mediapipe',
'The url prefix for links to code.') 'The url prefix for links to code.')
_SEARCH_HINTS = flags.DEFINE_bool( _SEARCH_HINTS = flags.DEFINE_bool(
'search_hints', True, 'search_hints', True,
'Include metadata search hints in the generated files') 'Include metadata search hints in the generated files')
_SITE_PATH = flags.DEFINE_string('site_path', '/mediapipe/api_docs/python', _SITE_PATH = flags.DEFINE_string('site_path', '/mediapipe/api/solutions/python',
'Path prefix in the _toc.yaml') 'Path prefix in the _toc.yaml')
@@ -0,0 +1,697 @@
---
layout: default
title: Building Graphs in C++
parent: Graphs
nav_order: 1
---
# Building Graphs in C++
{: .no_toc }
1. TOC
{:toc}
---
C++ graph builder is a powerful tool for:
* Building complex graphs
* Parametrizing graphs (e.g. setting a delegate on `InferenceCalculator`,
enabling/disabling parts of the graph)
* Deduplicating graphs (e.g. instead of CPU and GPU dedicated graphs in pbtxt
you can have a single code that constructs required graphs, sharing as much
as possible)
* Supporting optional graph inputs/outputs
* Customizing graphs per platform
## Basic Usage
Let's see how C++ graph builder can be used for a simple graph:
```proto
# Graph inputs.
input_stream: "input_tensors"
input_side_packet: "model"
# Graph outputs.
output_stream: "output_tensors"
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
input_side_packet: "MODEL:model"
output_stream: "TENSORS:output_tensors"
node_options: {
[type.googleapis.com/mediapipe.InferenceCalculatorOptions] {
# Requesting GPU delegate.
delegate { gpu {} }
}
}
}
```
Function to build the above `CalculatorGraphConfig` may look like:
```c++
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Graph inputs.
Stream<std::vector<Tensor>> input_tensors =
graph.In(0).SetName("input_tensors").Cast<std::vector<Tensor>>();
SidePacket<TfLiteModelPtr> model =
graph.SideIn(0).SetName("model").Cast<TfLiteModelPtr>();
auto& inference_node = graph.AddNode("InferenceCalculator");
auto& inference_opts =
inference_node.GetOptions<InferenceCalculatorOptions>();
// Requesting GPU delegate.
inference_opts.mutable_delegate()->mutable_gpu();
input_tensors.ConnectTo(inference_node.In("TENSORS"));
model.ConnectTo(inference_node.SideIn("MODEL"));
Stream<std::vector<Tensor>> output_tensors =
inference_node.Out("TENSORS").Cast<std::vector<Tensor>>();
// Graph outputs.
output_tensors.SetName("output_tensors").ConnectTo(graph.Out(0));
// Get `CalculatorGraphConfig` to pass it into `CalculatorGraph`
return graph.GetConfig();
}
```
Short summary:
* Use `Graph::In/SideIn` to get graph inputs as `Stream/SidePacket`
* Use `Node::Out/SideOut` to get node outputs as `Stream/SidePacket`
* Use `Stream/SidePacket::ConnectTo` to connect streams and side packets to
node inputs (`Node::In/SideIn`) and graph outputs (`Graph::Out/SideOut`)
* There's a "shortcut" operator `>>` that you can use instead of
`ConnectTo` function (E.g. `x >> node.In("IN")`).
* `Stream/SidePacket::Cast` is used to cast stream or side packet of `AnyType`
(E.g. `Stream<AnyType> in = graph.In(0);`) to a particular type
* Using actual types instead of `AnyType` sets you on a better path for
unleashing graph builder capabilities and improving your graphs
readability.
## Advanced Usage
### Utility Functions
Let's extract inference construction code into a dedicated utility function to
help for readability and code reuse:
```c++
// Updates graph to run inference.
Stream<std::vector<Tensor>> RunInference(
Stream<std::vector<Tensor>> tensors, SidePacket<TfLiteModelPtr> model,
const InferenceCalculatorOptions::Delegate& delegate, Graph& graph) {
auto& inference_node = graph.AddNode("InferenceCalculator");
auto& inference_opts =
inference_node.GetOptions<InferenceCalculatorOptions>();
*inference_opts.mutable_delegate() = delegate;
tensors.ConnectTo(inference_node.In("TENSORS"));
model.ConnectTo(inference_node.SideIn("MODEL"));
return inference_node.Out("TENSORS").Cast<std::vector<Tensor>>();
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Graph inputs.
Stream<std::vector<Tensor>> input_tensors =
graph.In(0).SetName("input_tensors").Cast<std::vector<Tensor>>();
SidePacket<TfLiteModelPtr> model =
graph.SideIn(0).SetName("model").Cast<TfLiteModelPtr>();
InferenceCalculatorOptions::Delegate delegate;
delegate.mutable_gpu();
Stream<std::vector<Tensor>> output_tensors =
RunInference(input_tensors, model, delegate, graph);
// Graph outputs.
output_tensors.SetName("output_tensors").ConnectTo(graph.Out(0));
return graph.GetConfig();
}
```
As a result, `RunInference` provides a clear interface stating what are the
inputs/outputs and their types.
It can be easily reused, e.g. it's only a few lines if you want to run an extra
model inference:
```c++
// Run first inference.
Stream<std::vector<Tensor>> output_tensors =
RunInference(input_tensors, model, delegate, graph);
// Run second inference on the output of the first one.
Stream<std::vector<Tensor>> extra_output_tensors =
RunInference(output_tensors, extra_model, delegate, graph);
```
And you don't need to duplicate names and tags (`InferenceCalculator`,
`TENSORS`, `MODEL`) or introduce dedicated constants here and there - those
details are localized to `RunInference` function.
Tip: extracting `RunInference` and similar functions to dedicated modules (e.g.
inference.h/cc which depends on the inference calculator) enables reuse in
graphs construction code and helps automatically pull in calculator dependencies
(e.g. no need to manually add `:inference_calculator` dep, just let your IDE
include `inference.h` and build cleaner pull in corresponding dependency).
### Utility Classes
And surely, it's not only about functions, in some cases it's beneficial to
introduce utility classes which can help making your graph construction code
more readable and less error prone.
MediaPipe offers `PassThroughCalculator` calculator, which is simply passing
through its inputs:
```
input_stream: "float_value"
input_stream: "int_value"
input_stream: "bool_value"
output_stream: "passed_float_value"
output_stream: "passed_int_value"
output_stream: "passed_bool_value"
node {
calculator: "PassThroughCalculator"
input_stream: "float_value"
input_stream: "int_value"
input_stream: "bool_value"
# The order must be the same as for inputs (or you can use explicit indexes)
output_stream: "passed_float_value"
output_stream: "passed_int_value"
output_stream: "passed_bool_value"
}
```
Let's see the straightforward C++ construction code to create the above graph:
```c++
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Graph inputs.
Stream<float> float_value = graph.In(0).SetName("float_value").Cast<float>();
Stream<int> int_value = graph.In(1).SetName("int_value").Cast<int>();
Stream<bool> bool_value = graph.In(2).SetName("bool_value").Cast<bool>();
auto& pass_node = graph.AddNode("PassThroughCalculator");
float_value.ConnectTo(pass_node.In("")[0]);
int_value.ConnectTo(pass_node.In("")[1]);
bool_value.ConnectTo(pass_node.In("")[2]);
Stream<float> passed_float_value = pass_node.Out("")[0].Cast<float>();
Stream<int> passed_int_value = pass_node.Out("")[1].Cast<int>();
Stream<bool> passed_bool_value = pass_node.Out("")[2].Cast<bool>();
// Graph outputs.
passed_float_value.SetName("passed_float_value").ConnectTo(graph.Out(0));
passed_int_value.SetName("passed_int_value").ConnectTo(graph.Out(1));
passed_bool_value.SetName("passed_bool_value").ConnectTo(graph.Out(2));
// Get `CalculatorGraphConfig` to pass it into `CalculatorGraph`
return graph.GetConfig();
}
```
While `pbtxt` representation maybe error prone (when we have many inputs to pass
through), C++ code looks even worse: repeated empty tags and `Cast` calls. Let's
see how we can do better by introducing a `PassThroughNodeBuilder`:
```c++
class PassThroughNodeBuilder {
public:
explicit PassThroughNodeBuilder(Graph& graph)
: node_(graph.AddNode("PassThroughCalculator")) {}
template <typename T>
Stream<T> PassThrough(Stream<T> stream) {
stream.ConnectTo(node_.In(index_));
return node_.Out(index_++).Cast<T>();
}
private:
int index_ = 0;
GenericNode& node_;
};
```
And now graph construction code can look like:
```c++
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Graph inputs.
Stream<float> float_value = graph.In(0).SetName("float_value").Cast<float>();
Stream<int> int_value = graph.In(1).SetName("int_value").Cast<int>();
Stream<bool> bool_value = graph.In(2).SetName("bool_value").Cast<bool>();
PassThroughNodeBuilder pass_node_builder(graph);
Stream<float> passed_float_value = pass_node_builder.PassThrough(float_value);
Stream<int> passed_int_value = pass_node_builder.PassThrough(int_value);
Stream<bool> passed_bool_value = pass_node_builder.PassThrough(bool_value);
// Graph outputs.
passed_float_value.SetName("passed_float_value").ConnectTo(graph.Out(0));
passed_int_value.SetName("passed_int_value").ConnectTo(graph.Out(1));
passed_bool_value.SetName("passed_bool_value").ConnectTo(graph.Out(2));
// Get `CalculatorGraphConfig` to pass it into `CalculatorGraph`
return graph.GetConfig();
}
```
Now you can't have incorrect order or index in your pass through construction
code and save some typing by guessing the type for `Cast` from the `PassThrough`
input.
Tip: the same as for the `RunInference` function, extracting
`PassThroughNodeBuilder` and similar utility classes into dedicated modules
enables reuse in graph construction code and helps to automatically pull in the
corresponding calculator dependencies.
## Dos and Don'ts
### Define graph inputs at the very beginning if possible
```c++ {.bad}
Stream<D> RunSomething(Stream<A> a, Stream<B> b, Graph& graph) {
Stream<C> c = graph.In(2).SetName("c").Cast<C>(); // Bad.
// ...
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
Stream<A> a = graph.In(0).SetName("a").Cast<A>();
// 10/100/N lines of code.
Stream<B> b = graph.In(1).SetName("b").Cast<B>() // Bad.
Stream<D> d = RunSomething(a, b, graph);
// ...
return graph.GetConfig();
}
```
In the above code:
* It can be hard to guess how many inputs you have in the graph.
* Can be error prone overall and hard to maintain in future (e.g. is it a
correct index? name? what if some inputs are removed or made optional?
etc.).
* `RunSomething` reuse is limited because other graphs may have different
inputs
Instead, define your graph inputs at the very beginning of your graph builder:
```c++ {.good}
Stream<D> RunSomething(Stream<A> a, Stream<B> b, Stream<C> c, Graph& graph) {
// ...
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).SetName("a").Cast<A>();
Stream<B> b = graph.In(1).SetName("b").Cast<B>();
Stream<C> c = graph.In(2).SetName("c").Cast<C>();
// 10/100/N lines of code.
Stream<D> d = RunSomething(a, b, c, graph);
// ...
return graph.GetConfig();
}
```
Use `std::optional` if you have an input stream or side packet that is not
always defined and put it at the very beginning:
```c++ {.good}
std::optional<Stream<A>> a;
if (needs_a) {
a = graph.In(0).SetName(a).Cast<A>();
}
```
Note: of course, there can be exceptions - for example, there can be a use case
where calling `RunSomething1(..., graph)`, ..., `RunSomethingN(..., graph)` is
**intended to add new inputs**, so afterwards you can iterate over them and feed
only added inputs into the graph. However, in any case, try to make it easy for
readers to find out what graph inputs it has or may have.
### Define graph outputs at the very end
```c++ {.bad}
void RunSomething(Stream<Input> input, Graph& graph) {
// ...
node.Out("OUTPUT_F")
.SetName("output_f").ConnectTo(graph.Out(2)); // Bad.
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// 10/100/N lines of code.
node.Out("OUTPUT_D")
.SetName("output_d").ConnectTo(graph.Out(0)); // Bad.
// 10/100/N lines of code.
node.Out("OUTPUT_E")
.SetName("output_e").ConnectTo(graph.Out(1)); // Bad.
// 10/100/N lines of code.
RunSomething(input, graph);
// ...
return graph.GetConfig();
}
```
In the above code:
* It can be hard to guess how many outputs you have in the graph.
* Can be error prone overall and hard to maintain in future (e.g. is it a
correct index? name? what if some outpus are removed or made optional?
etc.).
* `RunSomething` reuse is limited as other graphs may have different outputs
Instead, define your graph outputs at the very end of your graph builder:
```c++ {.good}
Stream<F> RunSomething(Stream<Input> input, Graph& graph) {
// ...
return node.Out("OUTPUT_F").Cast<F>();
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// 10/100/N lines of code.
Stream<D> d = node.Out("OUTPUT_D").Cast<D>();
// 10/100/N lines of code.
Stream<E> e = node.Out("OUTPUT_E").Cast<E>();
// 10/100/N lines of code.
Stream<F> f = RunSomething(input, graph);
// ...
// Outputs.
d.SetName("output_d").ConnectTo(graph.Out(0));
e.SetName("output_e").ConnectTo(graph.Out(1));
f.SetName("output_f").ConnectTo(graph.Out(2));
return graph.GetConfig();
}
```
### Keep nodes decoupled from each other
In MediaPipe, packet streams and side packets are as meaningful as processing
nodes. And any node input requirements and output products are expressed clearly
and independently in terms of the streams and side packets it consumes and
produces.
```c++ {.bad}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
auto& node1 = graph.AddNode("Calculator1");
a.ConnectTo(node1.In("INPUT"));
auto& node2 = graph.AddNode("Calculator2");
node1.Out("OUTPUT").ConnectTo(node2.In("INPUT")); // Bad.
auto& node3 = graph.AddNode("Calculator3");
node1.Out("OUTPUT").ConnectTo(node3.In("INPUT_B")); // Bad.
node2.Out("OUTPUT").ConnectTo(node3.In("INPUT_C")); // Bad.
auto& node4 = graph.AddNode("Calculator4");
node1.Out("OUTPUT").ConnectTo(node4.In("INPUT_B")); // Bad.
node2.Out("OUTPUT").ConnectTo(node4.In("INPUT_C")); // Bad.
node3.Out("OUTPUT").ConnectTo(node4.In("INPUT_D")); // Bad.
// Outputs.
node1.Out("OUTPUT").SetName("b").ConnectTo(graph.Out(0)); // Bad.
node2.Out("OUTPUT").SetName("c").ConnectTo(graph.Out(1)); // Bad.
node3.Out("OUTPUT").SetName("d").ConnectTo(graph.Out(2)); // Bad.
node4.Out("OUTPUT").SetName("e").ConnectTo(graph.Out(3)); // Bad.
return graph.GetConfig();
}
```
In the above code:
* Nodes are coupled to each other, e.g. `node4` knows where its inputs are
coming from (`node1`, `node2`, `node3`) and it complicates refactoring,
maintenance and code reuse
* Such usage pattern is a downgrade from proto representation, where nodes
are decoupled by default.
* `node#.Out("OUTPUT")` calls are duplicated and readability suffers as you
could use cleaner names instead and also provide an actual type.
So, to fix the above issues you can write the following graph construction code:
```c++ {.good}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
// `node1` usage is limited to 3 lines below.
auto& node1 = graph.AddNode("Calculator1");
a.ConnectTo(node1.In("INPUT"));
Stream<B> b = node1.Out("OUTPUT").Cast<B>();
// `node2` usage is limited to 3 lines below.
auto& node2 = graph.AddNode("Calculator2");
b.ConnectTo(node2.In("INPUT"));
Stream<C> c = node2.Out("OUTPUT").Cast<C>();
// `node3` usage is limited to 4 lines below.
auto& node3 = graph.AddNode("Calculator3");
b.ConnectTo(node3.In("INPUT_B"));
c.ConnectTo(node3.In("INPUT_C"));
Stream<D> d = node3.Out("OUTPUT").Cast<D>();
// `node4` usage is limited to 5 lines below.
auto& node4 = graph.AddNode("Calculator4");
b.ConnectTo(node4.In("INPUT_B"));
c.ConnectTo(node4.In("INPUT_C"));
d.ConnectTo(node4.In("INPUT_D"));
Stream<E> e = node4.Out("OUTPUT").Cast<E>();
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}
```
Now, if needed, you can easily remove `node1` and make `b` a graph input and no
updates are needed to `node2`, `node3`, `node4` (same as in proto representation
by the way), because they are decoupled from each other.
Overall, the above code replicates the proto graph more closely:
```proto
input_stream: "a"
node {
calculator: "Calculator1"
input_stream: "INPUT:a"
output_stream: "OUTPUT:b"
}
node {
calculator: "Calculator2"
input_stream: "INPUT:b"
output_stream: "OUTPUT:C"
}
node {
calculator: "Calculator3"
input_stream: "INPUT_B:b"
input_stream: "INPUT_C:c"
output_stream: "OUTPUT:d"
}
node {
calculator: "Calculator4"
input_stream: "INPUT_B:b"
input_stream: "INPUT_C:c"
input_stream: "INPUT_D:d"
output_stream: "OUTPUT:e"
}
output_stream: "b"
output_stream: "c"
output_stream: "d"
output_stream: "e"
```
On top of that, now you can extract utility functions for further reuse in other graphs:
```c++ {.good}
Stream<B> RunCalculator1(Stream<A> a, Graph& graph) {
auto& node = graph.AddNode("Calculator1");
a.ConnectTo(node.In("INPUT"));
return node.Out("OUTPUT").Cast<B>();
}
Stream<C> RunCalculator2(Stream<B> b, Graph& graph) {
auto& node = graph.AddNode("Calculator2");
b.ConnectTo(node.In("INPUT"));
return node.Out("OUTPUT").Cast<C>();
}
Stream<D> RunCalculator3(Stream<B> b, Stream<C> c, Graph& graph) {
auto& node = graph.AddNode("Calculator3");
b.ConnectTo(node.In("INPUT_B"));
c.ConnectTo(node.In("INPUT_C"));
return node.Out("OUTPUT").Cast<D>();
}
Stream<E> RunCalculator4(Stream<B> b, Stream<C> c, Stream<D> d, Graph& graph) {
auto& node = graph.AddNode("Calculator4");
b.ConnectTo(node.In("INPUT_B"));
c.ConnectTo(node.In("INPUT_C"));
d.ConnectTo(node.In("INPUT_D"));
return node.Out("OUTPUT").Cast<E>();
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
Stream<B> b = RunCalculator1(a, graph);
Stream<C> c = RunCalculator2(b, graph);
Stream<D> d = RunCalculator3(b, c, graph);
Stream<E> e = RunCalculator4(b, c, d, graph);
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}
```
### Separate nodes for better readability
```c++ {.bad}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
auto& node1 = graph.AddNode("Calculator1");
a.ConnectTo(node1.In("INPUT"));
Stream<B> b = node1.Out("OUTPUT").Cast<B>();
auto& node2 = graph.AddNode("Calculator2");
b.ConnectTo(node2.In("INPUT"));
Stream<C> c = node2.Out("OUTPUT").Cast<C>();
auto& node3 = graph.AddNode("Calculator3");
b.ConnectTo(node3.In("INPUT_B"));
c.ConnectTo(node3.In("INPUT_C"));
Stream<D> d = node3.Out("OUTPUT").Cast<D>();
auto& node4 = graph.AddNode("Calculator4");
b.ConnectTo(node4.In("INPUT_B"));
c.ConnectTo(node4.In("INPUT_C"));
d.ConnectTo(node4.In("INPUT_D"));
Stream<E> e = node4.Out("OUTPUT").Cast<E>();
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}
```
In the above code, it can be hard to grasp the idea where each node begins and
ends. To improve this and help your code readers, you can simply have blank
lines before and after each node:
```c++ {.good}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
auto& node1 = graph.AddNode("Calculator1");
a.ConnectTo(node1.In("INPUT"));
Stream<B> b = node1.Out("OUTPUT").Cast<B>();
auto& node2 = graph.AddNode("Calculator2");
b.ConnectTo(node2.In("INPUT"));
Stream<C> c = node2.Out("OUTPUT").Cast<C>();
auto& node3 = graph.AddNode("Calculator3");
b.ConnectTo(node3.In("INPUT_B"));
c.ConnectTo(node3.In("INPUT_C"));
Stream<D> d = node3.Out("OUTPUT").Cast<D>();
auto& node4 = graph.AddNode("Calculator4");
b.ConnectTo(node4.In("INPUT_B"));
c.ConnectTo(node4.In("INPUT_C"));
d.ConnectTo(node4.In("INPUT_D"));
Stream<E> e = node4.Out("OUTPUT").Cast<E>();
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}
```
Also, the above representation matches `CalculatorGraphConfig` proto
representation better.
If you extract nodes into utility functions, they are scoped within functions
already and it's clear where they begin and end, so it's completely fine to
have:
```c++ {.good}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
Stream<B> b = RunCalculator1(a, graph);
Stream<C> c = RunCalculator2(b, graph);
Stream<D> d = RunCalculator3(b, c, graph);
Stream<E> e = RunCalculator4(b, c, d, graph);
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}
```
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/framework_concepts/calculators
title: Calculators title: Calculators
parent: Framework Concepts parent: Framework Concepts
nav_order: 1 nav_order: 1
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/framework_concepts/overview
title: Framework Concepts title: Framework Concepts
nav_order: 5 nav_order: 5
has_children: true has_children: true
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/framework_concepts/gpu
title: GPU title: GPU
parent: Framework Concepts parent: Framework Concepts
nav_order: 5 nav_order: 5
+50 -20
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/framework_concepts/graphs
title: Graphs title: Graphs
parent: Framework Concepts parent: Framework Concepts
nav_order: 2 nav_order: 2
@@ -12,31 +13,31 @@ nav_order: 2
{:toc} {:toc}
--- ---
## GraphConfig ## Graph
A `GraphConfig` is a specification that describes the topology and functionality A `CalculatorGraphConfig` proto specifies the topology and functionality of a
of a MediaPipe graph. In the specification, a node in the graph represents an MediaPipe graph. Each `node` in the graph represents a particular calculator or
instance of a particular calculator. All the necessary configurations of the subgraph, and specifies necessary configurations, such as registered
node, such its type, inputs and outputs must be described in the specification. calculator/subgraph type, inputs, outputs and optional fields, such as
Description of the node can also include several optional fields, such as
node-specific options, input policy and executor, discussed in node-specific options, input policy and executor, discussed in
[Synchronization](synchronization.md). [Synchronization](synchronization.md).
`GraphConfig` has several other fields to configure the global graph-level `CalculatorGraphConfig` has several other fields to configure global graph-level
settings, eg, graph executor configs, number of threads, and maximum queue size settings, e.g. graph executor configs, number of threads, and maximum queue size
of input streams. Several graph-level settings are useful for tuning the of input streams. Several graph-level settings are useful for tuning the
performance of the graph on different platforms (eg, desktop v.s. mobile). For performance of the graph on different platforms (e.g., desktop v.s. mobile). For
instance, on mobile, attaching a heavy model-inference calculator to a separate instance, on mobile, attaching a heavy model-inference calculator to a separate
executor can improve the performance of a real-time application since this executor can improve the performance of a real-time application since this
enables thread locality. enables thread locality.
Below is a trivial `GraphConfig` example where we have series of passthrough Below is a trivial `CalculatorGraphConfig` example where we have series of
calculators : passthrough calculators :
```proto ```proto
# This graph named main_pass_throughcals_nosubgraph.pbtxt contains 4 # This graph named main_pass_throughcals_nosubgraph.pbtxt contains 4
# passthrough calculators. # passthrough calculators.
input_stream: "in" input_stream: "in"
output_stream: "out"
node { node {
calculator: "PassThroughCalculator" calculator: "PassThroughCalculator"
input_stream: "in" input_stream: "in"
@@ -55,17 +56,46 @@ node {
node { node {
calculator: "PassThroughCalculator" calculator: "PassThroughCalculator"
input_stream: "out3" input_stream: "out3"
output_stream: "out4" output_stream: "out"
} }
``` ```
MediaPipe offers an alternative `C++` representation for complex graphs (e.g. ML pipelines, handling model metadata, optional nodes, etc.). The above graph may look like:
```c++
CalculatorGraphConfig BuildGraphConfig() {
Graph graph;
// Graph inputs
Stream<AnyType> in = graph.In(0).SetName("in");
auto pass_through_fn = [](Stream<AnyType> in,
Graph& graph) -> Stream<AnyType> {
auto& node = graph.AddNode("PassThroughCalculator");
in.ConnectTo(node.In(0));
return node.Out(0);
};
Stream<AnyType> out1 = pass_through_fn(in, graph);
Stream<AnyType> out2 = pass_through_fn(out1, graph);
Stream<AnyType> out3 = pass_through_fn(out2, graph);
Stream<AnyType> out4 = pass_through_fn(out3, graph);
// Graph outputs
out4.SetName("out").ConnectTo(graph.Out(0));
return graph.GetConfig();
}
```
See more details in [Building Graphs in C++](building_graphs_cpp.md)
## Subgraph ## Subgraph
To modularize a `CalculatorGraphConfig` into sub-modules and assist with re-use To modularize a `CalculatorGraphConfig` into sub-modules and assist with re-use
of perception solutions, a MediaPipe graph can be defined as a `Subgraph`. The of perception solutions, a MediaPipe graph can be defined as a `Subgraph`. The
public interface of a subgraph consists of a set of input and output streams public interface of a subgraph consists of a set of input and output streams
similar to a calculator's public interface. The subgraph can then be included in similar to a calculator's public interface. The subgraph can then be included in
an `CalculatorGraphConfig` as if it were a calculator. When a MediaPipe graph is a `CalculatorGraphConfig` as if it were a calculator. When a MediaPipe graph is
loaded from a `CalculatorGraphConfig`, each subgraph node is replaced by the loaded from a `CalculatorGraphConfig`, each subgraph node is replaced by the
corresponding graph of calculators. As a result, the semantics and performance corresponding graph of calculators. As a result, the semantics and performance
of the subgraph is identical to the corresponding graph of calculators. of the subgraph is identical to the corresponding graph of calculators.
@@ -151,7 +181,7 @@ protobuf specified for a MediaPipe calculator. These "graph options" can be
specified where a graph is invoked, and used to populate calculator options and specified where a graph is invoked, and used to populate calculator options and
subgraph options within the graph. subgraph options within the graph.
In a CalculatorGraphConfig, graph options can be specified for a subgraph In a `CalculatorGraphConfig`, graph options can be specified for a subgraph
exactly like calculator options, as shown below: exactly like calculator options, as shown below:
``` ```
@@ -178,7 +208,7 @@ node {
} }
``` ```
In a CalculatorGraphConfig, graph options can be accepted and used to populate In a `CalculatorGraphConfig`, graph options can be accepted and used to populate
calculator options, as shown below: calculator options, as shown below:
``` ```
@@ -188,7 +218,7 @@ graph_options: {
node: { node: {
calculator: "ImageToTensorCalculator" calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image" input_stream: "IMAGE:image"
node_options: { node_options: {
[type.googleapis.com/mediapipe.ImageToTensorCalculatorOptions] { [type.googleapis.com/mediapipe.ImageToTensorCalculatorOptions] {
keep_aspect_ratio: true keep_aspect_ratio: true
@@ -247,9 +277,9 @@ NOTE: The current approach is experimental and subject to change. We welcome
your feedback. your feedback.
Please use the `CalculatorGraphTest.Cycle` unit test in Please use the `CalculatorGraphTest.Cycle` unit test in
`mediapipe/framework/calculator_graph_test.cc` as sample code. Shown `mediapipe/framework/calculator_graph_test.cc` as sample code. Shown below is
below is the cyclic graph in the test. The `sum` output of the adder is the sum the cyclic graph in the test. The `sum` output of the adder is the sum of the
of the integers generated by the integer source calculator. integers generated by the integer source calculator.
![a cyclic graph that adds a stream of integers](https://mediapipe.dev/images/cyclic_integer_sum_graph.svg "A cyclic graph") ![a cyclic graph that adds a stream of integers](https://mediapipe.dev/images/cyclic_integer_sum_graph.svg "A cyclic graph")
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/framework_concepts/packets
title: Packets title: Packets
parent: Framework Concepts parent: Framework Concepts
nav_order: 3 nav_order: 3
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/framework_concepts/realtime_streams
title: Real-time Streams title: Real-time Streams
parent: Framework Concepts parent: Framework Concepts
nav_order: 6 nav_order: 6
+6 -5
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/framework_concepts/synchronization
title: Synchronization title: Synchronization
parent: Framework Concepts parent: Framework Concepts
nav_order: 4 nav_order: 4
@@ -112,14 +113,14 @@ Warning: On the other hand, it is not guaranteed that an input packet will
always be available for all streams. always be available for all streams.
To explain how it works, we need to introduce the definition of a settled To explain how it works, we need to introduce the definition of a settled
timestamp. We say that a timestamp in a stream is *settled* if it lower than the timestamp. We say that a timestamp in a stream is *settled* if it is lower than
timestamp bound. In other words, a timestamp is settled for a stream once the the timestamp bound. In other words, a timestamp is settled for a stream once
state of the input at that timestamp is irrevocably known: either there is a the state of the input at that timestamp is irrevocably known: either there is a
packet, or there is the certainty that a packet with that timestamp will not packet, or there is the certainty that a packet with that timestamp will not
arrive. arrive.
Note: For this reason, MediaPipe also allows a stream producer to explicitly Note: For this reason, MediaPipe also allows a stream producer to explicitly
advance the timestamp bound farther that what the last packet implies, i.e. to advance the timestamp bound farther than what the last packet implies, i.e. to
provide a tighter bound. This can allow the downstream nodes to settle their provide a tighter bound. This can allow the downstream nodes to settle their
inputs sooner. inputs sooner.
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/android
title: MediaPipe on Android title: MediaPipe on Android
parent: Getting Started parent: Getting Started
has_children: true has_children: true
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/android_archive_library
title: MediaPipe Android Archive title: MediaPipe Android Archive
parent: MediaPipe on Android parent: MediaPipe on Android
grand_parent: Getting Started grand_parent: Getting Started
+11
View File
@@ -13,6 +13,17 @@ nav_order: 2
{:toc} {:toc}
--- ---
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023. This content will not be moved to
the new site, but will remain available in the source code repository on an
as-is basis.*
*This notice and web page will be removed on April 3, 2023.*
----
MediaPipe Android Solution APIs (currently in alpha) are available in: MediaPipe Android Solution APIs (currently in alpha) are available in:
* [MediaPipe Face Detection](../solutions/face_detection#android-solution-api) * [MediaPipe Face Detection](../solutions/face_detection#android-solution-api)
+11
View File
@@ -12,6 +12,17 @@ nav_exclude: true
{:toc} {:toc}
--- ---
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023. This content will not be moved to
the new site, but will remain available in the source code repository on an
as-is basis.*
*This notice and web page will be removed on April 3, 2023.*
----
### Android ### Android
Please see these [instructions](./android.md). Please see these [instructions](./android.md).
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/cpp
title: MediaPipe in C++ title: MediaPipe in C++
parent: Getting Started parent: Getting Started
has_children: true has_children: true
+3 -2
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/faq
title: FAQ title: FAQ
parent: Getting Started parent: Getting Started
nav_order: 9 nav_order: 9
@@ -59,7 +60,7 @@ The second approach allows up to [`max_in_flight`] invocations of the
packets from [`CalculatorBase::Process`] are automatically ordered by timestamp packets from [`CalculatorBase::Process`] are automatically ordered by timestamp
before they are passed along to downstream calculators. before they are passed along to downstream calculators.
With either aproach, you must be aware that the calculator running in parallel With either approach, you must be aware that the calculator running in parallel
cannot maintain internal state in the same way as a normal sequential cannot maintain internal state in the same way as a normal sequential
calculator. calculator.
+11
View File
@@ -11,3 +11,14 @@ has_children: true
1. TOC 1. TOC
{:toc} {:toc}
--- ---
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023. This content will not be moved to
the new site, but will remain available in the source code repository on an
as-is basis.*
*This notice and web page will be removed on April 3, 2023.*
----
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/gpu_support
title: GPU Support title: GPU Support
parent: Getting Started parent: Getting Started
nav_order: 7 nav_order: 7
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/hello_world_android
title: Hello World! on Android title: Hello World! on Android
parent: MediaPipe on Android parent: MediaPipe on Android
grand_parent: Getting Started grand_parent: Getting Started
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/hello_world_cpp
title: Hello World! in C++ title: Hello World! in C++
parent: MediaPipe in C++ parent: MediaPipe in C++
grand_parent: Getting Started grand_parent: Getting Started
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/hello_world_ios
title: Hello World! on iOS title: Hello World! on iOS
parent: MediaPipe on iOS parent: MediaPipe on iOS
grand_parent: Getting Started grand_parent: Getting Started
+4 -3
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/help
title: Getting Help title: Getting Help
parent: Getting Started parent: Getting Started
nav_order: 8 nav_order: 8
@@ -37,8 +38,8 @@ If you open a GitHub issue, here is our policy:
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: - **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device**: - **Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device**:
- **Bazel version**: - **Bazel version**:
- **Android Studio, NDK, SDK versions (if issue is related to building in mobile dev enviroment)**: - **Android Studio, NDK, SDK versions (if issue is related to building in mobile dev environment)**:
- **Xcode & Tulsi version (if issue is related to building in mobile dev enviroment)**: - **Xcode & Tulsi version (if issue is related to building in mobile dev environment)**:
- **Exact steps to reproduce**: - **Exact steps to reproduce**:
### Describe the problem ### Describe the problem
+16 -8
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/install
title: Installation title: Installation
parent: Getting Started parent: Getting Started
nav_order: 6 nav_order: 6
@@ -35,7 +36,7 @@ install --user six`.
```bash ```bash
$ cd $HOME $ cd $HOME
$ git clone https://github.com/google/mediapipe.git $ git clone --depth 1 https://github.com/google/mediapipe.git
# Change directory into MediaPipe root directory # Change directory into MediaPipe root directory
$ cd mediapipe $ cd mediapipe
@@ -67,6 +68,13 @@ install --user six`.
libopencv-video-dev libopencv-video-dev
``` ```
**Note**. On Debian 11/Ubuntu 21.04 where OpenCV 4.5 is installed with
`libopencv-video-dev`, `libopencv-contrib-dev` should also be installed.
```bash
$ sudo apt-get install -y libopencv-contrib-dev
```
MediaPipe's [`opencv_linux.BUILD`] and [`WORKSPACE`] are already configured MediaPipe's [`opencv_linux.BUILD`] and [`WORKSPACE`] are already configured
for OpenCV 2/3 and should work correctly on any architecture: for OpenCV 2/3 and should work correctly on any architecture:
@@ -287,7 +295,7 @@ build issues.
2. Checkout MediaPipe repository. 2. Checkout MediaPipe repository.
```bash ```bash
$ git clone https://github.com/google/mediapipe.git $ git clone --depth 1 https://github.com/google/mediapipe.git
# Change directory into MediaPipe root directory # Change directory into MediaPipe root directory
$ cd mediapipe $ cd mediapipe
@@ -416,7 +424,7 @@ build issues.
3. Checkout MediaPipe repository. 3. Checkout MediaPipe repository.
```bash ```bash
$ git clone https://github.com/google/mediapipe.git $ git clone --depth 1 https://github.com/google/mediapipe.git
$ cd mediapipe $ cd mediapipe
``` ```
@@ -590,7 +598,7 @@ next section.
7. Checkout MediaPipe repository. 7. Checkout MediaPipe repository.
``` ```
C:\Users\Username\mediapipe_repo> git clone https://github.com/google/mediapipe.git C:\Users\Username\mediapipe_repo> git clone --depth 1 https://github.com/google/mediapipe.git
# Change directory into MediaPipe root directory # Change directory into MediaPipe root directory
C:\Users\Username\mediapipe_repo> cd mediapipe C:\Users\Username\mediapipe_repo> cd mediapipe
@@ -651,7 +659,7 @@ cameras. Alternatively, you use a video file as input.
1. Follow the 1. Follow the
[instruction](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to [instruction](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to
install Windows Sysystem for Linux (Ubuntu). install Windows Subsystem for Linux (Ubuntu).
2. Install Windows ADB and start the ADB server in Windows. 2. Install Windows ADB and start the ADB server in Windows.
@@ -680,7 +688,7 @@ cameras. Alternatively, you use a video file as input.
6. Checkout MediaPipe repository. 6. Checkout MediaPipe repository.
```bash ```bash
username@DESKTOP-TMVLBJ1:~$ git clone https://github.com/google/mediapipe.git username@DESKTOP-TMVLBJ1:~$ git clone --depth 1 https://github.com/google/mediapipe.git
username@DESKTOP-TMVLBJ1:~$ cd mediapipe username@DESKTOP-TMVLBJ1:~$ cd mediapipe
``` ```
@@ -771,7 +779,7 @@ This will use a Docker image that will isolate mediapipe's installation from the
2. Build a docker image with tag "mediapipe". 2. Build a docker image with tag "mediapipe".
```bash ```bash
$ git clone https://github.com/google/mediapipe.git $ git clone --depth 1 https://github.com/google/mediapipe.git
$ cd mediapipe $ cd mediapipe
$ docker build --tag=mediapipe . $ docker build --tag=mediapipe .
+6 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/ios
title: MediaPipe on iOS title: MediaPipe on iOS
parent: Getting Started parent: Getting Started
has_children: true has_children: true
@@ -91,6 +92,10 @@ allows you to make use of automatic provisioning (see later section).
This will install `Tulsi.app` inside the `Applications` directory in your This will install `Tulsi.app` inside the `Applications` directory in your
home directory. home directory.
**Note**: Please ensure the `xcode_version` in the
[`build_and_run.sh`](https://github.com/bazelbuild/tulsi/blob/b1d0108e6a93dbe8ab01529b2c607b6b651f0759/build_and_run.sh#L26)
file in tulsi repo is the same version as installed in your system.
2. Open `mediapipe/Mediapipe.tulsiproj` using the Tulsi app. 2. Open `mediapipe/Mediapipe.tulsiproj` using the Tulsi app.
Tip: If Tulsi displays an error saying "Bazel could not be found", press the Tip: If Tulsi displays an error saying "Bazel could not be found", press the
+13 -2
View File
@@ -12,6 +12,17 @@ nav_order: 4
{:toc} {:toc}
--- ---
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023. This content will not be moved to
the new site, but will remain available in the source code repository on an
as-is basis.*
*This notice and web page will be removed on April 3, 2023.*
----
## Ready-to-use JavaScript Solutions ## Ready-to-use JavaScript Solutions
MediaPipe currently offers the following solutions: MediaPipe currently offers the following solutions:
@@ -29,11 +40,11 @@ Solution | NPM Package | Example
Click on a solution link above for more information, including API and code Click on a solution link above for more information, including API and code
snippets. snippets.
### Supported plaforms: ### Supported platforms:
| Browser | Platform | Notes | | Browser | Platform | Notes |
| ------- | ----------------------- | -------------------------------------- | | ------- | ----------------------- | -------------------------------------- |
| Chrome | Android / Windows / Mac | Pixel 4 and older unsupported. Fuschia | | Chrome | Android / Windows / Mac | Pixel 4 and older unsupported. Fuchsia |
| | | unsupported. | | | | unsupported. |
| Chrome | iOS | Camera unavailable in Chrome on iOS. | | Chrome | iOS | Camera unavailable in Chrome on iOS. |
| Safari | iPad/iPhone/Mac | iOS and Safari on iPad / iPhone / | | Safari | iPad/iPhone/Mac | iOS and Safari on iPad / iPhone / |
+2 -2
View File
@@ -1,6 +1,6 @@
--- ---
layout: default layout: forward
title: MediaPipe Python Framework target: https://developers.google.com/mediapipe/framework/getting_started/python_framework
parent: MediaPipe in Python parent: MediaPipe in Python
grand_parent: Getting Started grand_parent: Getting Started
nav_order: 1 nav_order: 1
+3 -2
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/getting_started/troubleshooting
title: Troubleshooting title: Troubleshooting
parent: Getting Started parent: Getting Started
nav_order: 10 nav_order: 10
@@ -65,7 +66,7 @@ WARNING: Download from https://storage.googleapis.com/mirror.tensorflow.org/gith
``` ```
usually indicates that Bazel fails to download necessary dependency repositories usually indicates that Bazel fails to download necessary dependency repositories
that MediaPipe needs. MedaiPipe has several dependency repositories that are that MediaPipe needs. MediaPipe has several dependency repositories that are
hosted by Google sites. In some regions, you may need to set up a network proxy hosted by Google sites. In some regions, you may need to set up a network proxy
or use a VPN to access those resources. You may also need to append or use a VPN to access those resources. You may also need to append
`--host_jvm_args "-DsocksProxyHost=<ip address> -DsocksProxyPort=<port number>"` `--host_jvm_args "-DsocksProxyHost=<ip address> -DsocksProxyPort=<port number>"`
+11
View File
@@ -19,6 +19,17 @@ ML solutions for live and streaming media.
![ready_to_use.png](https://mediapipe.dev/images/ready_to_use_small.png) | ![open_source.png](https://mediapipe.dev/images/open_source_small.png) ![ready_to_use.png](https://mediapipe.dev/images/ready_to_use_small.png) | ![open_source.png](https://mediapipe.dev/images/open_source_small.png)
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable* ***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
----
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023.*
*This notice and web page will be removed on April 3, 2023.*
----
## ML solutions in MediaPipe ## ML solutions in MediaPipe
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
+10
View File
@@ -18,6 +18,16 @@ nav_order: 14
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
AutoFlip is an automatic video cropping pipeline built on top of MediaPipe. This AutoFlip is an automatic video cropping pipeline built on top of MediaPipe. This
+10
View File
@@ -18,6 +18,16 @@ nav_order: 10
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
MediaPipe Box Tracking has been powering real-time tracking in MediaPipe Box Tracking has been powering real-time tracking in
+34 -4
View File
@@ -18,6 +18,16 @@ nav_order: 1
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
MediaPipe Face Detection is an ultrafast face detection solution that comes with MediaPipe Face Detection is an ultrafast face detection solution that comes with
@@ -54,6 +64,25 @@ used for its improved inference speed. Please refer to the
[model cards](./models.md#face_detection) for details. Default to `0` if not [model cards](./models.md#face_detection) for details. Default to `0` if not
specified. specified.
Note: Not available for JavaScript (use "model" instead).
#### model
A string value to indicate which model should be used. Use "short" to
select a short-range model that works best for faces within 2 meters from the
camera, and "full" for a full-range model best for faces within 5 meters. For
the full-range option, a sparse model is used for its improved inference speed.
Please refer to the model cards for details. Default to empty string.
Note: Valid only for JavaScript solution.
#### selfie_mode
A boolean value to indicate whether to flip the images/video frames
horizontally or not. Default to `false`.
Note: Valid only for JavaScript solution.
#### min_detection_confidence #### min_detection_confidence
Minimum confidence value (`[0.0, 1.0]`) from the face detection model for the Minimum confidence value (`[0.0, 1.0]`) from the face detection model for the
@@ -146,9 +175,9 @@ Please first see general [introduction](../getting_started/javascript.md) on
MediaPipe in JavaScript, then learn more in the companion [web demo](#resources) MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
and the following usage example. and the following usage example.
Supported configuration options: Supported face detection options:
* [selfieMode](#selfie_mode)
* [modelSelection](#model_selection) * [model](#model)
* [minDetectionConfidence](#min_detection_confidence) * [minDetectionConfidence](#min_detection_confidence)
```html ```html
@@ -176,6 +205,7 @@ Supported configuration options:
const videoElement = document.getElementsByClassName('input_video')[0]; const videoElement = document.getElementsByClassName('input_video')[0];
const canvasElement = document.getElementsByClassName('output_canvas')[0]; const canvasElement = document.getElementsByClassName('output_canvas')[0];
const canvasCtx = canvasElement.getContext('2d'); const canvasCtx = canvasElement.getContext('2d');
const drawingUtils = window;
function onResults(results) { function onResults(results) {
// Draw the overlays. // Draw the overlays.
@@ -199,7 +229,7 @@ const faceDetection = new FaceDetection({locateFile: (file) => {
return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${file}`; return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${file}`;
}}); }});
faceDetection.setOptions({ faceDetection.setOptions({
modelSelection: 0, model: 'short',
minDetectionConfidence: 0.5 minDetectionConfidence: 0.5
}); });
faceDetection.onResults(onResults); faceDetection.onResults(onResults);
+11 -1
View File
@@ -18,6 +18,16 @@ nav_order: 2
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
MediaPipe Face Mesh is a solution that estimates 468 3D face landmarks in MediaPipe Face Mesh is a solution that estimates 468 3D face landmarks in
@@ -133,7 +143,7 @@ about the model in this [paper](https://arxiv.org/abs/2006.10962).
The [Face Landmark Model](#face-landmark-model) performs a single-camera face landmark The [Face Landmark Model](#face-landmark-model) performs a single-camera face landmark
detection in the screen coordinate space: the X- and Y- coordinates are detection in the screen coordinate space: the X- and Y- coordinates are
normalized screen coordinates, while the Z coordinate is relative and is scaled normalized screen coordinates, while the Z coordinate is relative and is scaled
as the X coodinate under the as the X coordinate under the
[weak perspective projection camera model](https://en.wikipedia.org/wiki/3D_projection#Weak_perspective_projection). [weak perspective projection camera model](https://en.wikipedia.org/wiki/3D_projection#Weak_perspective_projection).
This format is well-suited for some applications, however it does not directly This format is well-suited for some applications, however it does not directly
enable the full spectrum of augmented reality (AR) features like aligning a enable the full spectrum of augmented reality (AR) features like aligning a
+10
View File
@@ -18,6 +18,16 @@ nav_order: 8
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
![hair_segmentation_android_gpu_gif](https://mediapipe.dev/images/mobile/hair_segmentation_android_gpu.gif) ![hair_segmentation_android_gpu_gif](https://mediapipe.dev/images/mobile/hair_segmentation_android_gpu.gif)
## Example Apps ## Example Apps
+10
View File
@@ -18,6 +18,16 @@ nav_order: 4
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
The ability to perceive the shape and motion of hands can be a vital component The ability to perceive the shape and motion of hands can be a vital component
+11
View File
@@ -18,6 +18,16 @@ nav_order: 6
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
Live perception of simultaneous [human pose](./pose.md), Live perception of simultaneous [human pose](./pose.md),
@@ -259,6 +269,7 @@ mp_holistic = mp.solutions.holistic
# For static images: # For static images:
IMAGE_FILES = [] IMAGE_FILES = []
BG_COLOR = (192, 192, 192) # gray
with mp_holistic.Holistic( with mp_holistic.Holistic(
static_image_mode=True, static_image_mode=True,
model_complexity=2, model_complexity=2,
+10
View File
@@ -18,6 +18,16 @@ nav_order: 11
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
Augmented Reality (AR) technology creates fun, engaging, and immersive user Augmented Reality (AR) technology creates fun, engaging, and immersive user
+12 -2
View File
@@ -18,6 +18,16 @@ nav_order: 3
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
A wide range of real-world applications, including computational photography A wide range of real-world applications, including computational photography
@@ -38,7 +48,7 @@ camera, in real-time, without the need for specialized hardware. Through use of
iris landmarks, the solution is also able to determine the metric distance iris landmarks, the solution is also able to determine the metric distance
between the subject and the camera with relative error less than 10%. Note that between the subject and the camera with relative error less than 10%. Note that
iris tracking does not infer the location at which people are looking, nor does iris tracking does not infer the location at which people are looking, nor does
it provide any form of identity recognition. With the cross-platfrom capability it provide any form of identity recognition. With the cross-platform capability
of the MediaPipe framework, MediaPipe Iris can run on most modern of the MediaPipe framework, MediaPipe Iris can run on most modern
[mobile phones](#mobile), [desktops/laptops](#desktop) and even on the [mobile phones](#mobile), [desktops/laptops](#desktop) and even on the
[web](#web). [web](#web).
@@ -99,7 +109,7 @@ You can also find more details in this
### Iris Landmark Model ### Iris Landmark Model
The iris model takes an image patch of the eye region and estimates both the eye The iris model takes an image patch of the eye region and estimates both the eye
landmarks (along the eyelid) and iris landmarks (along ths iris contour). You landmarks (along the eyelid) and iris landmarks (along this iris contour). You
can find more details in this [paper](https://arxiv.org/abs/2006.11341). can find more details in this [paper](https://arxiv.org/abs/2006.11341).
![iris_tracking_eye_and_iris_landmarks.png](https://mediapipe.dev/images/mobile/iris_tracking_eye_and_iris_landmarks.png) | ![iris_tracking_eye_and_iris_landmarks.png](https://mediapipe.dev/images/mobile/iris_tracking_eye_and_iris_landmarks.png) |
+10
View File
@@ -18,6 +18,16 @@ nav_order: 13
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
MediaPipe KNIFT is a template-based feature matching solution using KNIFT MediaPipe KNIFT is a template-based feature matching solution using KNIFT
+11 -1
View File
@@ -18,6 +18,16 @@ nav_order: 15
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
MediaPipe is a useful and general framework for media processing that can MediaPipe is a useful and general framework for media processing that can
@@ -85,7 +95,7 @@ process new data sets, in the documentation of
MediaSequence uses SequenceExamples as the format of both inputs and MediaSequence uses SequenceExamples as the format of both inputs and
outputs. Annotations are encoded as inputs in a SequenceExample of metadata outputs. Annotations are encoded as inputs in a SequenceExample of metadata
that defines the labels and the path to the cooresponding video file. This that defines the labels and the path to the corresponding video file. This
metadata is passed as input to the C++ `media_sequence_demo` binary, and the metadata is passed as input to the C++ `media_sequence_demo` binary, and the
output is a SequenceExample filled with images and annotations ready for output is a SequenceExample filled with images and annotations ready for
model training. model training.
+14 -2
View File
@@ -12,6 +12,20 @@ nav_order: 30
{:toc} {:toc}
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for
[these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
as of March 1, 2023. All other
[MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
to a new MediaPipe Solution. The code repository and prebuilt binaries for all
MediaPipe Legacy Solutions will continue to be provided on an as-is basis.
We encourage you to check out the new MediaPipe Solutions at:
[https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
*This notice and web page will be removed on April 3, 2023.*
----
### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection) ### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
* Short-range model (best for faces within 2 meters from the camera): * Short-range model (best for faces within 2 meters from the camera):
@@ -94,8 +108,6 @@ one over the other.
* [TFLite model](https://storage.googleapis.com/mediapipe-assets/ssdlite_object_detection.tflite) * [TFLite model](https://storage.googleapis.com/mediapipe-assets/ssdlite_object_detection.tflite)
* [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite) * [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
* [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
* [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
### [Objectron](https://google.github.io/mediapipe/solutions/objectron) ### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
+10
View File
@@ -18,6 +18,16 @@ nav_order: 9
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
![object_detection_android_gpu.gif](https://mediapipe.dev/images/mobile/object_detection_android_gpu.gif) ![object_detection_android_gpu.gif](https://mediapipe.dev/images/mobile/object_detection_android_gpu.gif)
## Example Apps ## Example Apps
+12 -2
View File
@@ -18,6 +18,16 @@ nav_order: 12
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
MediaPipe Objectron is a mobile real-time 3D object detection solution for MediaPipe Objectron is a mobile real-time 3D object detection solution for
@@ -170,7 +180,7 @@ and a
The detection subgraph performs ML inference only once every few frames to The detection subgraph performs ML inference only once every few frames to
reduce computation load, and decodes the output tensor to a FrameAnnotation that reduce computation load, and decodes the output tensor to a FrameAnnotation that
contains nine keypoints: the 3D bounding box's center and its eight vertices. contains nine keypoints: the 3D bounding box's center and its eight vertices.
The tracking subgraph runs every frame, using the box traker in The tracking subgraph runs every frame, using the box tracker in
[MediaPipe Box Tracking](./box_tracking.md) to track the 2D box tightly [MediaPipe Box Tracking](./box_tracking.md) to track the 2D box tightly
enclosing the projection of the 3D bounding box, and lifts the tracked 2D enclosing the projection of the 3D bounding box, and lifts the tracked 2D
keypoints to 3D with keypoints to 3D with
@@ -613,7 +623,7 @@ z_ndc = 1 / Z
### Pixel Space ### Pixel Space
In this API we set upper-left coner of an image as the origin of pixel In this API we set upper-left corner of an image as the origin of pixel
coordinate. One can convert from NDC to pixel space as follows: coordinate. One can convert from NDC to pixel space as follows:
``` ```
+10
View File
@@ -20,6 +20,16 @@ nav_order: 5
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
Human pose estimation from video plays a critical role in various applications Human pose estimation from video plays a critical role in various applications
+10
View File
@@ -19,6 +19,16 @@ nav_order: 1
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
One of the applications One of the applications
+10
View File
@@ -18,6 +18,16 @@ nav_order: 7
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview ## Overview
*Fig 1. Example of MediaPipe Selfie Segmentation.* | *Fig 1. Example of MediaPipe Selfie Segmentation.* |
+16
View File
@@ -13,6 +13,22 @@ has_toc: false
{:toc} {:toc}
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions. We have
ended support for
[these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
as of March 1, 2023. All other
[MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
to a new MediaPipe Solution. The
[code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
and prebuilt binaries for all MediaPipe Legacy Solutions will continue to
be provided on an as-is basis. We encourage you to check out the new MediaPipe
Solutions at:
[https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
*This notice and web page will be removed on June 1, 2023.*
----
MediaPipe offers open source cross-platform, customizable ML solutions for live MediaPipe offers open source cross-platform, customizable ML solutions for live
and streaming media. and streaming media.
+10
View File
@@ -18,6 +18,16 @@ nav_order: 16
</details> </details>
--- ---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
MediaPipe is a useful and general framework for media processing that can assist MediaPipe is a useful and general framework for media processing that can assist
with research, development, and deployment of ML models. This example focuses on with research, development, and deployment of ML models. This example focuses on
model development by demonstrating how to prepare training data and do model model development by demonstrating how to prepare training data and do model
+2 -1
View File
@@ -1,5 +1,6 @@
--- ---
layout: default layout: forward
target: https://developers.google.com/mediapipe/framework/tools/visualizer
title: Visualizer title: Visualizer
parent: Tools parent: Tools
nav_order: 1 nav_order: 1
+2 -2
View File
@@ -12,12 +12,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"]) licenses(["notice"])
package(default_visibility = ["//visibility:private"]) package(default_visibility = ["//visibility:private"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library( proto_library(
name = "mfcc_mel_calculators_proto", name = "mfcc_mel_calculators_proto",
srcs = ["mfcc_mel_calculators.proto"], srcs = ["mfcc_mel_calculators.proto"],
@@ -280,6 +280,13 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_, audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
&window); &window);
break; break;
case SpectrogramCalculatorOptions::SQRT_HANN: {
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
&window);
absl::c_transform(window, window.begin(),
[](double x) { return std::sqrt(x); });
break;
}
} }
// Propagate settings down to the actual Spectrogram object. // Propagate settings down to the actual Spectrogram object.
@@ -68,6 +68,7 @@ message SpectrogramCalculatorOptions {
HANN = 0; HANN = 0;
HAMMING = 1; HAMMING = 1;
COSINE = 2; COSINE = 2;
SQRT_HANN = 4;
} }
optional WindowType window_type = 6 [default = HANN]; optional WindowType window_type = 6 [default = HANN];
+25 -9
View File
@@ -13,12 +13,21 @@
# limitations under the License. # limitations under the License.
# #
load("@bazel_skylib//lib:selects.bzl", "selects")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"]) licenses(["notice"])
package(default_visibility = ["//visibility:public"]) package(default_visibility = ["//visibility:public"])
selects.config_setting_group(
name = "ios_or_disable_gpu",
match_any = [
"//mediapipe/gpu:disable_gpu",
"//mediapipe:ios",
],
)
mediapipe_proto_library( mediapipe_proto_library(
name = "concatenate_vector_calculator_proto", name = "concatenate_vector_calculator_proto",
srcs = ["concatenate_vector_calculator.proto"], srcs = ["concatenate_vector_calculator.proto"],
@@ -108,6 +117,7 @@ mediapipe_proto_library(
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
"//mediapipe/framework/formats:classification_proto", "//mediapipe/framework/formats:classification_proto",
"//mediapipe/framework/formats:landmark_proto", "//mediapipe/framework/formats:landmark_proto",
"//mediapipe/framework/formats:time_series_header_proto",
], ],
) )
@@ -188,6 +198,7 @@ cc_library(
"//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
@@ -210,6 +221,7 @@ cc_library(
"//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
@@ -226,10 +238,12 @@ cc_test(
":begin_loop_calculator", ":begin_loop_calculator",
":end_loop_calculator", ":end_loop_calculator",
":gate_calculator", ":gate_calculator",
":pass_through_calculator",
"//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_contract", "//mediapipe/framework:calculator_contract",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet", "//mediapipe/framework:packet",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:parse_text_proto",
@@ -567,7 +581,7 @@ cc_library(
name = "packet_thinner_calculator", name = "packet_thinner_calculator",
srcs = ["packet_thinner_calculator.cc"], srcs = ["packet_thinner_calculator.cc"],
deps = [ deps = [
"//mediapipe/calculators/core:packet_thinner_calculator_cc_proto", ":packet_thinner_calculator_cc_proto",
"//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/formats:video_stream_header",
@@ -584,7 +598,7 @@ cc_test(
srcs = ["packet_thinner_calculator_test.cc"], srcs = ["packet_thinner_calculator_test.cc"],
deps = [ deps = [
":packet_thinner_calculator", ":packet_thinner_calculator",
"//mediapipe/calculators/core:packet_thinner_calculator_cc_proto", ":packet_thinner_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner", "//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/formats:video_stream_header",
@@ -762,7 +776,7 @@ cc_library(
srcs = ["packet_resampler_calculator.cc"], srcs = ["packet_resampler_calculator.cc"],
hdrs = ["packet_resampler_calculator.h"], hdrs = ["packet_resampler_calculator.h"],
deps = [ deps = [
"//mediapipe/calculators/core:packet_resampler_calculator_cc_proto", ":packet_resampler_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:collection_item_id", "//mediapipe/framework:collection_item_id",
"//mediapipe/framework/deps:mathutil", "//mediapipe/framework/deps:mathutil",
@@ -786,7 +800,7 @@ cc_test(
], ],
deps = [ deps = [
":packet_resampler_calculator", ":packet_resampler_calculator",
"//mediapipe/calculators/core:packet_resampler_calculator_cc_proto", ":packet_resampler_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner", "//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/formats:video_stream_header",
@@ -852,10 +866,10 @@ cc_test(
name = "flow_limiter_calculator_test", name = "flow_limiter_calculator_test",
srcs = ["flow_limiter_calculator_test.cc"], srcs = ["flow_limiter_calculator_test.cc"],
deps = [ deps = [
":counting_source_calculator",
":flow_limiter_calculator", ":flow_limiter_calculator",
":flow_limiter_calculator_cc_proto", ":flow_limiter_calculator_cc_proto",
"//mediapipe/calculators/core:counting_source_calculator", ":pass_through_calculator",
"//mediapipe/calculators/core:pass_through_calculator",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner", "//mediapipe/framework:calculator_runner",
"//mediapipe/framework:test_calculators", "//mediapipe/framework:test_calculators",
@@ -899,8 +913,7 @@ cc_library(
"@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops", "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], ":ios_or_disable_gpu": [],
"//mediapipe:ios": [],
"//conditions:default": [ "//conditions:default": [
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
], ],
@@ -1150,6 +1163,7 @@ cc_library(
"//mediapipe/framework:collection_item_id", "//mediapipe/framework:collection_item_id",
"//mediapipe/framework/formats:classification_cc_proto", "//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
@@ -1302,7 +1316,7 @@ cc_test(
srcs = ["packet_sequencer_calculator_test.cc"], srcs = ["packet_sequencer_calculator_test.cc"],
deps = [ deps = [
":packet_sequencer_calculator", ":packet_sequencer_calculator",
"//mediapipe/calculators/core:pass_through_calculator", ":pass_through_calculator",
"//mediapipe/framework:calculator_cc_proto", "//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:subgraph", "//mediapipe/framework:subgraph",
@@ -1321,8 +1335,10 @@ cc_library(
hdrs = ["merge_to_vector_calculator.h"], hdrs = ["merge_to_vector_calculator.h"],
deps = [ deps = [
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework/api2:node", "//mediapipe/framework/api2:node",
"//mediapipe/framework/api2:port", "//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:image", "//mediapipe/framework/formats:image",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
], ],
@@ -20,6 +20,7 @@
#include "mediapipe/calculators/core/end_loop_calculator.h" #include "mediapipe/calculators/core/end_loop_calculator.h"
#include "mediapipe/framework/calculator_contract.h" #include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/packet.h" #include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/gtest.h"
@@ -444,5 +445,67 @@ TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, MultipleVectors) {
PacketOfIntsEq(input_timestamp2, std::vector<int>{6, 9}))); PacketOfIntsEq(input_timestamp2, std::vector<int>{6, 9})));
} }
absl::Status InitBeginEndTensorLoopTestGraph(
CalculatorGraph& graph, std::vector<Packet>& output_packets) {
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
num_threads: 4
input_stream: "tensors"
node {
calculator: "BeginLoopTensorCalculator"
input_stream: "ITERABLE:tensors"
output_stream: "ITEM:tensor"
output_stream: "BATCH_END:timestamp"
}
node {
calculator: "PassThroughCalculator"
input_stream: "tensor"
output_stream: "passed_tensor"
}
node {
calculator: "EndLoopTensorCalculator"
input_stream: "ITEM:passed_tensor"
input_stream: "BATCH_END:timestamp"
output_stream: "ITERABLE:output_tensors"
}
)pb");
tool::AddVectorSink("output_tensors", &graph_config, &output_packets);
MP_RETURN_IF_ERROR(graph.Initialize(graph_config));
return graph.StartRun({});
}
TEST(BeginEndTensorLoopCalculatorGraphTest, SingleNonEmptyVector) {
// Initialize the graph.
CalculatorGraph graph;
std::vector<Packet> output_packets;
MP_ASSERT_OK(InitBeginEndTensorLoopTestGraph(graph, output_packets));
// Prepare the inputs and run.
Timestamp input_timestamp = Timestamp(0);
std::vector<mediapipe::Tensor> tensors;
for (int i = 0; i < 4; i++) {
tensors.emplace_back(Tensor::ElementType::kFloat32,
Tensor::Shape{4, 3, 2, 1});
}
Packet vector_packet =
MakePacket<std::vector<mediapipe::Tensor>>(std::move(tensors));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tensors", std::move(vector_packet).At(input_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
// Verify the output packet.
EXPECT_EQ(output_packets.size(), 1);
const std::vector<Tensor>& output_tensors =
output_packets[0].Get<std::vector<Tensor>>();
EXPECT_EQ(output_tensors.size(), 4);
for (int i = 0; i < output_tensors.size(); i++) {
EXPECT_THAT(output_tensors[i].shape().dims,
testing::ElementsAre(4, 3, 2, 1));
}
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_ASSERT_OK(graph.WaitUntilDone());
}
} // namespace } // namespace
} // namespace mediapipe } // namespace mediapipe
@@ -20,6 +20,7 @@
#include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h" #include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/formats/tensor.h"
namespace mediapipe { namespace mediapipe {
@@ -55,4 +56,8 @@ REGISTER_CALCULATOR(BeginLoopMatrixVectorCalculator);
typedef BeginLoopCalculator<std::vector<uint64_t>> BeginLoopUint64tCalculator; typedef BeginLoopCalculator<std::vector<uint64_t>> BeginLoopUint64tCalculator;
REGISTER_CALCULATOR(BeginLoopUint64tCalculator); REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
// A calculator to process std::vector<mediapipe::Tensor>.
typedef BeginLoopCalculator<std::vector<Tensor>> BeginLoopTensorCalculator;
REGISTER_CALCULATOR(BeginLoopTensorCalculator);
} // namespace mediapipe } // namespace mediapipe
@@ -15,7 +15,6 @@
#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_ #ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_ #define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
#include "absl/memory/memory.h"
#include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h" #include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -24,6 +23,7 @@
#include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
namespace mediapipe { namespace mediapipe {
@@ -49,7 +49,7 @@ namespace mediapipe {
// calculator: "EndLoopWithOutputCalculator" // calculator: "EndLoopWithOutputCalculator"
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts // input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts // input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
// output_stream: "OUTPUT:aggregated_result" # IterableU @ext_ts // output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
// } // }
// //
// Input streams tagged with "CLONE" are cloned to the corresponding output // Input streams tagged with "CLONE" are cloned to the corresponding output
@@ -112,13 +112,38 @@ class BeginLoopCalculator : public CalculatorBase {
absl::Status Process(CalculatorContext* cc) final { absl::Status Process(CalculatorContext* cc) final {
Timestamp last_timestamp = loop_internal_timestamp_; Timestamp last_timestamp = loop_internal_timestamp_;
if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) { if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) {
const IterableT& collection = // Try to consume the ITERABLE packet if possible to obtain the ownership
cc->Inputs().Tag("ITERABLE").template Get<IterableT>(); // and emit the item packets by moving them.
for (const auto& item : collection) { // If the ITERABLE packet is not consumable, then try to copy each item
cc->Outputs().Tag("ITEM").AddPacket( // instead. If the ITEM type is not copy constructible, an error will be
MakePacket<ItemT>(item).At(loop_internal_timestamp_)); // returned.
ForwardClonePackets(cc, loop_internal_timestamp_); auto iterable_ptr_or =
++loop_internal_timestamp_; cc->Inputs().Tag("ITERABLE").Value().Consume<IterableT>();
if (iterable_ptr_or.ok()) {
for (auto& item : *iterable_ptr_or.value()) {
Packet item_packet = MakePacket<ItemT>(std::move(item));
cc->Outputs().Tag("ITEM").AddPacket(
item_packet.At(loop_internal_timestamp_));
ForwardClonePackets(cc, loop_internal_timestamp_);
++loop_internal_timestamp_;
}
} else {
if constexpr (std::is_copy_constructible<ItemT>()) {
const IterableT& collection =
cc->Inputs().Tag("ITERABLE").template Get<IterableT>();
for (const auto& item : collection) {
cc->Outputs().Tag("ITEM").AddPacket(
MakePacket<ItemT>(item).At(loop_internal_timestamp_));
ForwardClonePackets(cc, loop_internal_timestamp_);
++loop_internal_timestamp_;
}
} else {
return absl::InternalError(
"The element type is not copiable. Consider making the "
"BeginLoopCalculator the sole owner of the input packet so that "
"the "
"items can be consumed and moved.");
}
} }
} }
@@ -138,7 +163,6 @@ class BeginLoopCalculator : public CalculatorBase {
.Tag("BATCH_END") .Tag("BATCH_END")
.AddPacket(MakePacket<Timestamp>(cc->InputTimestamp()) .AddPacket(MakePacket<Timestamp>(cc->InputTimestamp())
.At(Timestamp(loop_internal_timestamp_ - 1))); .At(Timestamp(loop_internal_timestamp_ - 1)));
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -19,6 +19,7 @@
#include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
@@ -84,6 +85,8 @@ class ConstantSidePacketCalculator : public CalculatorBase {
packet.Set<LandmarkList>(); packet.Set<LandmarkList>();
} else if (packet_options.has_double_value()) { } else if (packet_options.has_double_value()) {
packet.Set<double>(); packet.Set<double>();
} else if (packet_options.has_time_series_header_value()) {
packet.Set<TimeSeriesHeader>();
} else { } else {
return absl::InvalidArgumentError( return absl::InvalidArgumentError(
"None of supported values were specified in options."); "None of supported values were specified in options.");
@@ -118,6 +121,9 @@ class ConstantSidePacketCalculator : public CalculatorBase {
MakePacket<LandmarkList>(packet_options.landmark_list_value())); MakePacket<LandmarkList>(packet_options.landmark_list_value()));
} else if (packet_options.has_double_value()) { } else if (packet_options.has_double_value()) {
packet.Set(MakePacket<double>(packet_options.double_value())); packet.Set(MakePacket<double>(packet_options.double_value()));
} else if (packet_options.has_time_series_header_value()) {
packet.Set(MakePacket<TimeSeriesHeader>(
packet_options.time_series_header_value()));
} else { } else {
return absl::InvalidArgumentError( return absl::InvalidArgumentError(
"None of supported values were specified in options."); "None of supported values were specified in options.");
@@ -19,6 +19,7 @@ package mediapipe;
import "mediapipe/framework/calculator.proto"; import "mediapipe/framework/calculator.proto";
import "mediapipe/framework/formats/classification.proto"; import "mediapipe/framework/formats/classification.proto";
import "mediapipe/framework/formats/landmark.proto"; import "mediapipe/framework/formats/landmark.proto";
import "mediapipe/framework/formats/time_series_header.proto";
message ConstantSidePacketCalculatorOptions { message ConstantSidePacketCalculatorOptions {
extend CalculatorOptions { extend CalculatorOptions {
@@ -35,6 +36,7 @@ message ConstantSidePacketCalculatorOptions {
ClassificationList classification_list_value = 6; ClassificationList classification_list_value = 6;
LandmarkList landmark_list_value = 7; LandmarkList landmark_list_value = 7;
double double_value = 9; double double_value = 9;
TimeSeriesHeader time_series_header_value = 10;
} }
} }
@@ -21,6 +21,7 @@
#include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h" #include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/util/render_data.pb.h" #include "mediapipe/util/render_data.pb.h"
#include "tensorflow/lite/interpreter.h" #include "tensorflow/lite/interpreter.h"
@@ -52,8 +53,9 @@ typedef EndLoopCalculator<std::vector<::mediapipe::ClassificationList>>
EndLoopClassificationListCalculator; EndLoopClassificationListCalculator;
REGISTER_CALCULATOR(EndLoopClassificationListCalculator); REGISTER_CALCULATOR(EndLoopClassificationListCalculator);
typedef EndLoopCalculator<std::vector<TfLiteTensor>> EndLoopTensorCalculator; typedef EndLoopCalculator<std::vector<TfLiteTensor>>
REGISTER_CALCULATOR(EndLoopTensorCalculator); EndLoopTfLiteTensorCalculator;
REGISTER_CALCULATOR(EndLoopTfLiteTensorCalculator);
typedef EndLoopCalculator<std::vector<::mediapipe::Detection>> typedef EndLoopCalculator<std::vector<::mediapipe::Detection>>
EndLoopDetectionCalculator; EndLoopDetectionCalculator;
@@ -62,4 +64,7 @@ REGISTER_CALCULATOR(EndLoopDetectionCalculator);
typedef EndLoopCalculator<std::vector<Matrix>> EndLoopMatrixCalculator; typedef EndLoopCalculator<std::vector<Matrix>> EndLoopMatrixCalculator;
REGISTER_CALCULATOR(EndLoopMatrixCalculator); REGISTER_CALCULATOR(EndLoopMatrixCalculator);
typedef EndLoopCalculator<std::vector<Tensor>> EndLoopTensorCalculator;
REGISTER_CALCULATOR(EndLoopTensorCalculator);
} // namespace mediapipe } // namespace mediapipe
@@ -15,6 +15,8 @@
#ifndef MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_ #ifndef MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_ #define MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
#include <type_traits>
#include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h" #include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -75,8 +77,23 @@ class EndLoopCalculator : public CalculatorBase {
if (!input_stream_collection_) { if (!input_stream_collection_) {
input_stream_collection_.reset(new IterableT); input_stream_collection_.reset(new IterableT);
} }
input_stream_collection_->push_back( // Try to consume the item and move it into the collection. If the items
cc->Inputs().Tag("ITEM").template Get<ItemT>()); // are not consumable, then try to copy them instead. If the items are
// not copiable, then an error will be returned.
auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
if (item_ptr_or.ok()) {
input_stream_collection_->push_back(std::move(*item_ptr_or.value()));
} else {
if constexpr (std::is_copy_constructible_v<ItemT>) {
input_stream_collection_->push_back(
cc->Inputs().Tag("ITEM").template Get<ItemT>());
} else {
return absl::InternalError(
"The item type is not copiable. Consider making the "
"EndLoopCalculator the sole owner of the input packets so that "
"it can be moved instead of copying.");
}
}
} }
if (!cc->Inputs().Tag("BATCH_END").Value().IsEmpty()) { // flush signal if (!cc->Inputs().Tag("BATCH_END").Value().IsEmpty()) { // flush signal
@@ -35,7 +35,6 @@ message FlowLimiterCalculatorOptions {
optional int32 max_in_queue = 2 [default = 0]; optional int32 max_in_queue = 2 [default = 0];
// The maximum time in microseconds to wait for a frame to finish processing. // The maximum time in microseconds to wait for a frame to finish processing.
// The default value stops waiting after 1 sec. // The default value 0 specifies no timeout.
// The value 0 specifies no timeout. optional int64 in_flight_timeout = 3 [default = 0];
optional int64 in_flight_timeout = 3 [default = 1000000];
} }
@@ -47,7 +47,7 @@ namespace api2 {
// calculator: "Get{SpecificType}VectorItemCalculator" // calculator: "Get{SpecificType}VectorItemCalculator"
// input_stream: "VECTOR:vector" // input_stream: "VECTOR:vector"
// input_stream: "INDEX:index" // input_stream: "INDEX:index"
// input_stream: "ITEM:item" // output_stream: "ITEM:item"
// options { // options {
// [mediapipe.GetVectorItemCalculatorOptions.ext] { // [mediapipe.GetVectorItemCalculatorOptions.ext] {
// item_index: 5 // item_index: 5
@@ -15,6 +15,7 @@ limitations under the License.
#include "mediapipe/calculators/core/merge_to_vector_calculator.h" #include "mediapipe/calculators/core/merge_to_vector_calculator.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image.h"
namespace mediapipe { namespace mediapipe {
@@ -27,5 +28,9 @@ typedef MergeToVectorCalculator<mediapipe::GpuBuffer>
MergeGpuBuffersToVectorCalculator; MergeGpuBuffersToVectorCalculator;
MEDIAPIPE_REGISTER_NODE(MergeGpuBuffersToVectorCalculator); MEDIAPIPE_REGISTER_NODE(MergeGpuBuffersToVectorCalculator);
typedef MergeToVectorCalculator<mediapipe::Detection>
MergeDetectionsToVectorCalculator;
MEDIAPIPE_REGISTER_NODE(MergeDetectionsToVectorCalculator);
} // namespace api2 } // namespace api2
} // namespace mediapipe } // namespace mediapipe
@@ -48,7 +48,6 @@ class MergeToVectorCalculator : public Node {
} }
absl::Status Process(CalculatorContext* cc) { absl::Status Process(CalculatorContext* cc) {
const int input_num = kIn(cc).Count();
std::vector<T> output_vector; std::vector<T> output_vector;
for (auto it = kIn(cc).begin(); it != kIn(cc).end(); it++) { for (auto it = kIn(cc).begin(); it != kIn(cc).end(); it++) {
const auto& elem = *it; const auto& elem = *it;
@@ -76,7 +76,11 @@ constexpr char kMaxInFlightTag[] = "MAX_IN_FLIGHT";
// } // }
// output_stream: "gated_frames" // output_stream: "gated_frames"
// } // }
class RealTimeFlowLimiterCalculator : public CalculatorBase { //
// Please use FlowLimiterCalculator, which replaces this calculator and
// defines a few additional configuration options.
class ABSL_DEPRECATED("Use FlowLimiterCalculator instead.")
RealTimeFlowLimiterCalculator : public CalculatorBase {
public: public:
static absl::Status GetContract(CalculatorContract* cc) { static absl::Status GetContract(CalculatorContract* cc) {
int num_data_streams = cc->Inputs().NumEntries(""); int num_data_streams = cc->Inputs().NumEntries("");
@@ -66,12 +66,16 @@ class SequenceShiftCalculator : public Node {
// The number of packets or timestamps we need to store to output packet[i] at // The number of packets or timestamps we need to store to output packet[i] at
// the timestamp of packet[i + packet_offset]; equal to abs(packet_offset). // the timestamp of packet[i + packet_offset]; equal to abs(packet_offset).
int cache_size_; int cache_size_;
bool emit_empty_packets_before_first_packet_ = false;
}; };
MEDIAPIPE_REGISTER_NODE(SequenceShiftCalculator); MEDIAPIPE_REGISTER_NODE(SequenceShiftCalculator);
absl::Status SequenceShiftCalculator::Open(CalculatorContext* cc) { absl::Status SequenceShiftCalculator::Open(CalculatorContext* cc) {
packet_offset_ = kOffset(cc).GetOr( packet_offset_ = kOffset(cc).GetOr(
cc->Options<mediapipe::SequenceShiftCalculatorOptions>().packet_offset()); cc->Options<mediapipe::SequenceShiftCalculatorOptions>().packet_offset());
emit_empty_packets_before_first_packet_ =
cc->Options<mediapipe::SequenceShiftCalculatorOptions>()
.emit_empty_packets_before_first_packet();
cache_size_ = abs(packet_offset_); cache_size_ = abs(packet_offset_);
// An offset of zero is a no-op, but someone might still request it. // An offset of zero is a no-op, but someone might still request it.
if (packet_offset_ == 0) { if (packet_offset_ == 0) {
@@ -96,6 +100,8 @@ void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
// Ready to output oldest packet with current timestamp. // Ready to output oldest packet with current timestamp.
kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp())); kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
packet_cache_.pop_front(); packet_cache_.pop_front();
} else if (emit_empty_packets_before_first_packet_) {
LOG(FATAL) << "Not supported yet";
} }
// Store current packet for later output. // Store current packet for later output.
packet_cache_.push_back(kIn(cc).packet()); packet_cache_.push_back(kIn(cc).packet());
@@ -23,4 +23,8 @@ message SequenceShiftCalculatorOptions {
optional SequenceShiftCalculatorOptions ext = 107633927; optional SequenceShiftCalculatorOptions ext = 107633927;
} }
optional int32 packet_offset = 1 [default = -1]; optional int32 packet_offset = 1 [default = -1];
// Emits empty packets before the first delayed packet is emitted. Takes
// effect only when packet offset is set to positive.
optional bool emit_empty_packets_before_first_packet = 2 [default = false];
} }
+8 -2
View File
@@ -301,12 +301,18 @@ cc_test(
":image_cropping_calculator", ":image_cropping_calculator",
":image_cropping_calculator_cc_proto", ":image_cropping_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/tool:tag_map", "//mediapipe/framework/tool:tag_map",
"//mediapipe/framework/tool:tag_map_helper", "//mediapipe/framework/tool:tag_map_helper",
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
], ],
) )
@@ -378,8 +384,8 @@ cc_library(
name = "scale_image_calculator", name = "scale_image_calculator",
srcs = ["scale_image_calculator.cc"], srcs = ["scale_image_calculator.cc"],
deps = [ deps = [
":scale_image_calculator_cc_proto",
":scale_image_utils", ":scale_image_utils",
"//mediapipe/calculators/image:scale_image_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_format_cc_proto", "//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
@@ -747,8 +753,8 @@ cc_test(
tags = ["desktop_only_test"], tags = ["desktop_only_test"],
deps = [ deps = [
":affine_transformation", ":affine_transformation",
":image_transformation_calculator",
":warp_affine_calculator", ":warp_affine_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_converter", "//mediapipe/calculators/tensor:image_to_tensor_converter",
"//mediapipe/calculators/tensor:image_to_tensor_utils", "//mediapipe/calculators/tensor:image_to_tensor_utils",
"//mediapipe/calculators/util:from_image_calculator", "//mediapipe/calculators/util:from_image_calculator",
@@ -92,8 +92,8 @@ class GlTextureWarpAffineRunner
constexpr GLchar kVertShader[] = R"( constexpr GLchar kVertShader[] = R"(
in vec4 position; in vec4 position;
in mediump vec4 texture_coordinate; in highp vec4 texture_coordinate;
out mediump vec2 sample_coordinate; out highp vec2 sample_coordinate;
uniform mat4 transform_matrix; uniform mat4 transform_matrix;
void main() { void main() {
@@ -104,7 +104,7 @@ class GlTextureWarpAffineRunner
)"; )";
constexpr GLchar kFragShader[] = R"( constexpr GLchar kFragShader[] = R"(
DEFAULT_PRECISION(mediump, float) DEFAULT_PRECISION(highp, float)
in vec2 sample_coordinate; in vec2 sample_coordinate;
uniform sampler2D input_texture; uniform sampler2D input_texture;
@@ -38,6 +38,7 @@ void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
constexpr char kRgbaInTag[] = "RGBA_IN"; constexpr char kRgbaInTag[] = "RGBA_IN";
constexpr char kRgbInTag[] = "RGB_IN"; constexpr char kRgbInTag[] = "RGB_IN";
constexpr char kBgrInTag[] = "BGR_IN";
constexpr char kBgraInTag[] = "BGRA_IN"; constexpr char kBgraInTag[] = "BGRA_IN";
constexpr char kGrayInTag[] = "GRAY_IN"; constexpr char kGrayInTag[] = "GRAY_IN";
constexpr char kRgbaOutTag[] = "RGBA_OUT"; constexpr char kRgbaOutTag[] = "RGBA_OUT";
@@ -57,6 +58,7 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
// RGB -> RGBA // RGB -> RGBA
// RGBA -> BGRA // RGBA -> BGRA
// BGRA -> RGBA // BGRA -> RGBA
// BGR -> RGB
// //
// This calculator only supports a single input stream and output stream at a // This calculator only supports a single input stream and output stream at a
// time. If more than one input stream or output stream is present, the // time. If more than one input stream or output stream is present, the
@@ -69,6 +71,7 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
// RGB_IN: The input video stream (ImageFrame, SRGB). // RGB_IN: The input video stream (ImageFrame, SRGB).
// BGRA_IN: The input video stream (ImageFrame, SBGRA). // BGRA_IN: The input video stream (ImageFrame, SBGRA).
// GRAY_IN: The input video stream (ImageFrame, GRAY8). // GRAY_IN: The input video stream (ImageFrame, GRAY8).
// BGR_IN: The input video stream (ImageFrame, SBGR).
// //
// Output streams: // Output streams:
// RGBA_OUT: The output video stream (ImageFrame, SRGBA). // RGBA_OUT: The output video stream (ImageFrame, SRGBA).
@@ -122,6 +125,10 @@ absl::Status ColorConvertCalculator::GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kBgraInTag).Set<ImageFrame>(); cc->Inputs().Tag(kBgraInTag).Set<ImageFrame>();
} }
if (cc->Inputs().HasTag(kBgrInTag)) {
cc->Inputs().Tag(kBgrInTag).Set<ImageFrame>();
}
if (cc->Outputs().HasTag(kRgbOutTag)) { if (cc->Outputs().HasTag(kRgbOutTag)) {
cc->Outputs().Tag(kRgbOutTag).Set<ImageFrame>(); cc->Outputs().Tag(kRgbOutTag).Set<ImageFrame>();
} }
@@ -194,6 +201,11 @@ absl::Status ColorConvertCalculator::Process(CalculatorContext* cc) {
return ConvertAndOutput(kRgbaInTag, kBgraOutTag, ImageFormat::SBGRA, return ConvertAndOutput(kRgbaInTag, kBgraOutTag, ImageFormat::SBGRA,
cv::COLOR_RGBA2BGRA, cc); cv::COLOR_RGBA2BGRA, cc);
} }
// BGR -> RGB
if (cc->Inputs().HasTag(kBgrInTag) && cc->Outputs().HasTag(kRgbOutTag)) {
return ConvertAndOutput(kBgrInTag, kRgbOutTag, ImageFormat::SRGB,
cv::COLOR_BGR2RGB, cc);
}
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Unsupported image format conversion."; << "Unsupported image format conversion.";
@@ -37,7 +37,8 @@ enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
namespace mediapipe { namespace mediapipe {
namespace { namespace {
using ::mediapipe::NormalizedRect;
using ::mediapipe::Rect;
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
@@ -222,8 +223,8 @@ absl::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
cv::Mat input_mat = formats::MatView(&input_img); cv::Mat input_mat = formats::MatView(&input_img);
RectSpec specs = GetCropSpecs(cc, input_img.Width(), input_img.Height()); RectSpec specs = GetCropSpecs(cc, input_img.Width(), input_img.Height());
int target_width = specs.width, target_height = specs.height, int target_width = specs.width, target_height = specs.height;
rect_center_x = specs.center_x, rect_center_y = specs.center_y; float rect_center_x = specs.center_x, rect_center_y = specs.center_y;
float rotation = specs.rotation; float rotation = specs.rotation;
// Get border mode and value for OpenCV. // Get border mode and value for OpenCV.
@@ -243,19 +244,29 @@ absl::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
output_width *= scale; output_width *= scale;
output_height *= scale; output_height *= scale;
float dst_corners[8] = {0, float dst_corners[8] = {
output_height - 1, 0, output_height, 0, 0, output_width, 0, output_width, output_height};
0, const cv::Mat dst_points = cv::Mat(4, 2, CV_32F, dst_corners);
0, // The projection matrix is computed using the corners of rects, not the
output_width - 1, // centers of corner pixels
0, const cv::Mat projection_matrix =
output_width - 1,
output_height - 1};
cv::Mat dst_points = cv::Mat(4, 2, CV_32F, dst_corners);
cv::Mat projection_matrix =
cv::getPerspectiveTransform(src_points, dst_points); cv::getPerspectiveTransform(src_points, dst_points);
// The projection matrix need to be adjusted because `cv::warpPerspective` is
// based on integer centers.
// clang-format off
double shift_src_vec[9] = {1.0, 0.0, 0.5,
0.0, 1.0, 0.5,
0.0, 0.0, 1.0};
double shift_dst_vec[9] = {1.0, 0.0, -0.5,
0.0, 1.0, -0.5,
0.0, 0.0, 1.0};
// clang-format on
const cv::Mat shift_src = cv::Mat(3, 3, CV_64F, shift_src_vec);
const cv::Mat shift_dst = cv::Mat(3, 3, CV_64F, shift_dst_vec);
const cv::Mat adjusted_projection_matrix =
shift_dst * projection_matrix * shift_src;
cv::Mat cropped_image; cv::Mat cropped_image;
cv::warpPerspective(input_mat, cropped_image, projection_matrix, cv::warpPerspective(input_mat, cropped_image, adjusted_projection_matrix,
cv::Size(output_width, output_height), cv::Size(output_width, output_height),
/* flags = */ 0, /* flags = */ 0,
/* borderMode = */ border_mode); /* borderMode = */ border_mode);
@@ -415,8 +426,8 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
int* dst_width, int* dst_width,
int* dst_height) { int* dst_height) {
RectSpec specs = GetCropSpecs(cc, src_width, src_height); RectSpec specs = GetCropSpecs(cc, src_width, src_height);
int crop_width = specs.width, crop_height = specs.height, int crop_width = specs.width, crop_height = specs.height;
x_center = specs.center_x, y_center = specs.center_y; float x_center = specs.center_x, y_center = specs.center_y;
float rotation = specs.rotation; float rotation = specs.rotation;
const float half_width = crop_width / 2.0f; const float half_width = crop_width / 2.0f;
@@ -465,8 +476,8 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
int crop_width = src_width; int crop_width = src_width;
int crop_height = src_height; int crop_height = src_height;
// Get the center of cropping box. Default is the at the center. // Get the center of cropping box. Default is the at the center.
int x_center = src_width / 2; float x_center = src_width / 2.0f;
int y_center = src_height / 2; float y_center = src_height / 2.0f;
// Get the rotation of the cropping box. // Get the rotation of the cropping box.
float rotation = 0.0f; float rotation = 0.0f;
// Get the normalized width and height if specified by the inputs or options. // Get the normalized width and height if specified by the inputs or options.
@@ -493,8 +504,8 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
if (norm_rect.width() > 0.0 && norm_rect.height() > 0.0) { if (norm_rect.width() > 0.0 && norm_rect.height() > 0.0) {
normalized_width = norm_rect.width(); normalized_width = norm_rect.width();
normalized_height = norm_rect.height(); normalized_height = norm_rect.height();
x_center = std::round(norm_rect.x_center() * src_width); x_center = norm_rect.x_center() * src_width;
y_center = std::round(norm_rect.y_center() * src_height); y_center = norm_rect.y_center() * src_height;
rotation = norm_rect.rotation(); rotation = norm_rect.rotation();
} }
} else if (cc->Inputs().HasTag(kWidthTag) && } else if (cc->Inputs().HasTag(kWidthTag) &&
@@ -520,14 +531,30 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
// present from the inputs. // present from the inputs.
if (!cc->Inputs().HasTag(kRectTag) && !cc->Inputs().HasTag(kNormRectTag)) { if (!cc->Inputs().HasTag(kRectTag) && !cc->Inputs().HasTag(kNormRectTag)) {
if (options.has_norm_center_x() && options.has_norm_center_y()) { if (options.has_norm_center_x() && options.has_norm_center_y()) {
x_center = std::round(options.norm_center_x() * src_width); x_center = options.norm_center_x() * src_width;
y_center = std::round(options.norm_center_y() * src_height); y_center = options.norm_center_y() * src_height;
} }
if (options.has_rotation()) { if (options.has_rotation()) {
rotation = options.rotation(); rotation = options.rotation();
} }
} }
if (rotation == 0.0f) {
// Adjust the center to the closest integer when the crop size is
// even-number and to the closest half-integer when the crop size is
// odd-number.
if (crop_width % 2 == 0) {
x_center = std::round(x_center);
} else {
x_center = std::round(x_center + 0.5f) - 0.5f;
}
if (crop_height % 2 == 0) {
y_center = std::round(y_center);
} else {
y_center = std::round(y_center + 0.5f) - 0.5f;
}
}
return {crop_width, crop_height, x_center, y_center, rotation}; return {crop_width, crop_height, x_center, y_center, rotation};
} }
@@ -42,8 +42,8 @@ namespace mediapipe {
struct RectSpec { struct RectSpec {
int width; int width;
int height; int height;
int center_x; float center_x;
int center_y; float center_y;
float rotation; float rotation;
bool operator==(const RectSpec& rect) const { bool operator==(const RectSpec& rect) const {
@@ -19,8 +19,12 @@
#include "mediapipe/calculators/image/image_cropping_calculator.pb.h" #include "mediapipe/calculators/image/image_cropping_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/rect.pb.h" #include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h" #include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/tag_map.h" #include "mediapipe/framework/tool/tag_map.h"
@@ -37,6 +41,166 @@ constexpr char kRectTag[] = "RECT";
constexpr char kHeightTag[] = "HEIGHT"; constexpr char kHeightTag[] = "HEIGHT";
constexpr char kWidthTag[] = "WIDTH"; constexpr char kWidthTag[] = "WIDTH";
std::unique_ptr<mediapipe::ImageFrame> GetInputFrame(int width, int height,
int channel) {
const int total_size = width * height * channel;
auto image_format = channel == 4 ? mediapipe::ImageFormat::SRGBA
: mediapipe::ImageFormat::SRGB;
auto input_frame = std::make_unique<mediapipe::ImageFrame>(
image_format, width, height, /*alignment_boundary =*/1);
for (int i = 0; i < total_size; ++i) {
input_frame->MutablePixelData()[i] = i % 256;
}
return input_frame;
}
// Test identity function, where cropping size is same as input size
TEST(ImageCroppingCalculatorTest, IdentityFunctionCropWithOriginalSize) {
auto calculator_node =
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
absl::Substitute(
R"pb(
calculator: "ImageCroppingCalculator"
input_stream: "IMAGE:input_frames"
output_stream: "IMAGE:cropped_output_frames"
options: {
[mediapipe.ImageCroppingCalculatorOptions.ext] {
width: $0
height: $1
}
}
)pb",
input_width, input_height));
mediapipe::CalculatorRunner runner(calculator_node);
// Input frame.
const auto input_frame = GetInputFrame(input_width, input_height, 3);
auto input_frame_packet =
mediapipe::MakePacket<mediapipe::ImageFrame>(std::move(*input_frame));
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
input_frame_packet.At(mediapipe::Timestamp(1)));
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
EXPECT_EQ(outputs.NumEntries(), 1);
const auto& output_image =
outputs.Tag("IMAGE").packets[0].Get<mediapipe::ImageFrame>();
const auto expected_output = GetInputFrame(input_width, input_height, 3);
cv::Mat output_mat = formats::MatView(&output_image);
cv::Mat expected_mat = formats::MatView(expected_output.get());
double max_diff = cv::norm(expected_mat, output_mat, cv::NORM_INF);
EXPECT_EQ(max_diff, 0);
} // TEST
// Test identity function, where cropping size is same as input size.
// When an image has an odd number for its size, its center falls on a
// fractional pixel. As a result, the values for center_x and center_y need to
// be of type float.
TEST(ImageCroppingCalculatorTest, IdentityFunctionCropWithOddSize) {
const int input_width = 99;
const int input_height = 99;
auto calculator_node =
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
absl::Substitute(
R"pb(
calculator: "ImageCroppingCalculator"
input_stream: "IMAGE:input_frames"
output_stream: "IMAGE:cropped_output_frames"
options: {
[mediapipe.ImageCroppingCalculatorOptions.ext] {
width: $0
height: $1
}
}
)pb",
input_width, input_height));
mediapipe::CalculatorRunner runner(calculator_node);
// Input frame.
const auto input_frame = GetInputFrame(input_width, input_height, 3);
auto input_frame_packet =
mediapipe::MakePacket<mediapipe::ImageFrame>(std::move(*input_frame));
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
input_frame_packet.At(mediapipe::Timestamp(1)));
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
EXPECT_EQ(outputs.NumEntries(), 1);
const auto& output_image =
outputs.Tag("IMAGE").packets[0].Get<mediapipe::ImageFrame>();
const auto expected_output = GetInputFrame(input_width, input_height, 3);
cv::Mat output_mat = formats::MatView(&output_image);
cv::Mat expected_mat = formats::MatView(expected_output.get());
double max_diff = cv::norm(expected_mat, output_mat, cv::NORM_INF);
EXPECT_EQ(max_diff, 0);
} // TEST
// Test identity function on GPU, where cropping size is same as input size.
TEST(ImageCroppingCalculatorTest, IdentityFunctionCropWithOriginalSizeGPU) {
mediapipe::CalculatorGraphConfig config =
ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(absl::Substitute(
R"pb(
input_stream: "input_frames"
node {
calculator: "ImageFrameToGpuBufferCalculator"
input_stream: "input_frames"
output_stream: "input_frames_gpu"
}
node {
calculator: "ImageCroppingCalculator"
input_stream: "IMAGE_GPU:input_frames_gpu"
output_stream: "IMAGE_GPU:cropped_output_frames_gpu"
options: {
[mediapipe.ImageCroppingCalculatorOptions.ext] {
width: $0
height: $1
}
}
}
node {
calculator: "GpuBufferToImageFrameCalculator"
input_stream: "cropped_output_frames_gpu"
output_stream: "cropped_output_frames"
}
)pb",
input_width, input_height));
std::vector<Packet> output_packets;
tool::AddVectorSink("cropped_output_frames", &config, &output_packets);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(config));
// Input frame.
const auto input_frame = GetInputFrame(input_width, input_height, 4);
auto input_frame_packet =
mediapipe::MakePacket<mediapipe::ImageFrame>(std::move(*input_frame));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_frames", input_frame_packet.At(mediapipe::Timestamp(1))));
MP_ASSERT_OK(graph.WaitUntilIdle());
// Get and process results.
const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
std::cout << output_image.Width();
const auto expected_output = GetInputFrame(input_width, input_height, 4);
cv::Mat output_mat = formats::MatView(&output_image);
cv::Mat expected_mat = formats::MatView(expected_output.get());
double max_diff = cv::norm(expected_mat, output_mat, cv::NORM_INF);
EXPECT_EQ(max_diff, 0);
} // TEST
// Test normal case, where norm_width and norm_height in options are set. // Test normal case, where norm_width and norm_height in options are set.
TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) { TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
auto calculator_node = auto calculator_node =
@@ -195,11 +359,11 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
auto cc = absl::make_unique<CalculatorContext>( auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), inputTags, tool::CreateTagMap({}).value()); calculator_state.get(), inputTags, tool::CreateTagMap({}).value());
auto& inputs = cc->Inputs(); auto& inputs = cc->Inputs();
mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>( Rect rect = ParseTextProtoOrDie<Rect>(
R"pb( R"pb(
width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5 width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
)pb"); )pb");
inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect); inputs.Tag(kRectTag).Value() = MakePacket<Rect>(rect);
RectSpec expectRect = { RectSpec expectRect = {
.width = 1, .width = 1,
.height = 1, .height = 1,
@@ -207,6 +207,7 @@ class ImageTransformationCalculator : public CalculatorBase {
bool flip_vertically_ = false; bool flip_vertically_ = false;
bool use_gpu_ = false; bool use_gpu_ = false;
cv::Scalar padding_color_;
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
GlCalculatorHelper gpu_helper_; GlCalculatorHelper gpu_helper_;
std::unique_ptr<QuadRenderer> rgb_renderer_; std::unique_ptr<QuadRenderer> rgb_renderer_;
@@ -338,6 +339,9 @@ absl::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
} }
scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE); scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE);
padding_color_ = cv::Scalar(options_.padding_color().red(),
options_.padding_color().green(),
options_.padding_color().blue());
if (use_gpu_) { if (use_gpu_) {
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
@@ -480,7 +484,8 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
cv::copyMakeBorder(intermediate_mat, scaled_mat, top, bottom, left, cv::copyMakeBorder(intermediate_mat, scaled_mat, top, bottom, left,
right, right,
options_.constant_padding() ? cv::BORDER_CONSTANT options_.constant_padding() ? cv::BORDER_CONSTANT
: cv::BORDER_REPLICATE); : cv::BORDER_REPLICATE,
padding_color_);
} else { } else {
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height), cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
0, 0, scale_flag); 0, 0, scale_flag);
@@ -20,11 +20,21 @@ import "mediapipe/calculators/image/rotation_mode.proto";
import "mediapipe/framework/calculator.proto"; import "mediapipe/framework/calculator.proto";
import "mediapipe/gpu/scale_mode.proto"; import "mediapipe/gpu/scale_mode.proto";
option java_package = "com.google.mediapipe.calculator.proto";
option java_outer_classname = "ImageTransformationCalculatorOptionsProto";
message ImageTransformationCalculatorOptions { message ImageTransformationCalculatorOptions {
extend CalculatorOptions { extend CalculatorOptions {
optional ImageTransformationCalculatorOptions ext = 251952830; optional ImageTransformationCalculatorOptions ext = 251952830;
} }
// RGB values in range of 0 - 255
message Color {
optional int32 red = 1 [default = 0];
optional int32 green = 2 [default = 0];
optional int32 blue = 3 [default = 0];
}
// Output dimensions. Set to 0 if they should be the same as the input. // Output dimensions. Set to 0 if they should be the same as the input.
optional int32 output_width = 1 [default = 0]; optional int32 output_width = 1 [default = 0];
optional int32 output_height = 2 [default = 0]; optional int32 output_height = 2 [default = 0];
@@ -40,4 +50,8 @@ message ImageTransformationCalculatorOptions {
// Default is to use BORDER_CONSTANT. If set to false, it will use // Default is to use BORDER_CONSTANT. If set to false, it will use
// BORDER_REPLICATE instead. // BORDER_REPLICATE instead.
optional bool constant_padding = 7 [default = true]; optional bool constant_padding = 7 [default = true];
// The color for the padding. This option is only used when the scale mode is
// FIT. Default is black. This is for CPU only.
optional Color padding_color = 8;
} }
@@ -142,6 +142,9 @@ absl::Status FindOutputDimensions(int input_width, //
static_cast<double>(input_height)); static_cast<double>(input_height));
try_width = (try_width / 2) * 2; try_width = (try_width / 2) * 2;
try_height = (try_height / 2) * 2; try_height = (try_height / 2) * 2;
// The output width/height should be greater than 0.
try_width = std::max(try_width, 1);
try_height = std::max(try_height, 1);
if (target_height <= 0 || try_height <= target_height) { if (target_height <= 0 || try_height <= target_height) {
// The resulting height based on the target width and aspect ratio // The resulting height based on the target width and aspect ratio
@@ -160,6 +163,9 @@ absl::Status FindOutputDimensions(int input_width, //
static_cast<double>(input_width)); static_cast<double>(input_width));
try_width = (try_width / 2) * 2; try_width = (try_width / 2) * 2;
try_height = (try_height / 2) * 2; try_height = (try_height / 2) * 2;
// The output width/height should be greater than 0.
try_width = std::max(try_width, 1);
try_height = std::max(try_height, 1);
if (target_width <= 0 || try_width <= target_width) { if (target_width <= 0 || try_width <= target_width) {
// The resulting width based on the target width and aspect ratio // The resulting width based on the target width and aspect ratio
@@ -124,6 +124,16 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
&output_width, &output_height)); &output_width, &output_height));
EXPECT_EQ(151, output_width); EXPECT_EQ(151, output_width);
EXPECT_EQ(101, output_height); EXPECT_EQ(101, output_height);
// Scale to height 1.
MP_ASSERT_OK(FindOutputDimensions(10000, 10, 100, 0, 0, true, 2,
&output_width, &output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(1, output_height);
// Scale to width 1.
MP_ASSERT_OK(FindOutputDimensions(10, 10000, 0, 100, 0, true, 2,
&output_width, &output_height));
EXPECT_EQ(1, output_width);
EXPECT_EQ(100, output_height);
} }
// Tests scaling without keeping the aspect ratio fixed. // Tests scaling without keeping the aspect ratio fixed.
@@ -47,10 +47,11 @@ constexpr int kNumChannelsRGBA = 4;
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
// Combines an RGB cv::Mat and a single-channel alpha cv::Mat of the same // Combines an RGB cv::Mat and an alpha cv::Mat of the same dimensions into an
// dimensions into an RGBA cv::Mat. Alpha may be read as uint8 or as another // RGBA cv::Mat. Alpha may be read as uint8 or as another numeric type; in the
// numeric type; in the latter case, it is upscaled to values between 0 and 255 // latter case, it is upscaled to values between 0 and 255 from an assumed input
// from an assumed input range of [0, 1). RGB and RGBA Mat's must be uchar. // range of [0, 1). Only the first channel of Alpha is used. Input & output Mat
// must be uchar.
template <typename AlphaType> template <typename AlphaType>
absl::Status MergeRGBA8Image(const cv::Mat input_mat, const cv::Mat& alpha_mat, absl::Status MergeRGBA8Image(const cv::Mat input_mat, const cv::Mat& alpha_mat,
cv::Mat& output_mat) { cv::Mat& output_mat) {
@@ -71,9 +72,9 @@ absl::Status MergeRGBA8Image(const cv::Mat input_mat, const cv::Mat& alpha_mat,
out_ptr[out_idx + 1] = in_ptr[in_idx + 1]; out_ptr[out_idx + 1] = in_ptr[in_idx + 1];
out_ptr[out_idx + 2] = in_ptr[in_idx + 2]; out_ptr[out_idx + 2] = in_ptr[in_idx + 2];
if constexpr (std::is_same<AlphaType, uchar>::value) { if constexpr (std::is_same<AlphaType, uchar>::value) {
out_ptr[out_idx + 3] = alpha_ptr[alpha_idx + 0]; out_ptr[out_idx + 3] = alpha_ptr[alpha_idx + 0]; // channel 0 of mask
} else { } else {
const AlphaType alpha = alpha_ptr[alpha_idx + 0]; const AlphaType alpha = alpha_ptr[alpha_idx + 0]; // channel 0 of mask
out_ptr[out_idx + 3] = static_cast<uchar>(round(alpha * 255.0f)); out_ptr[out_idx + 3] = static_cast<uchar>(round(alpha * 255.0f));
} }
} }
@@ -291,8 +292,8 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
const auto& alpha_mask = cc->Inputs().Tag(kInputAlphaTag).Get<ImageFrame>(); const auto& alpha_mask = cc->Inputs().Tag(kInputAlphaTag).Get<ImageFrame>();
cv::Mat alpha_mat = mediapipe::formats::MatView(&alpha_mask); cv::Mat alpha_mat = mediapipe::formats::MatView(&alpha_mask);
const bool alpha_is_float = alpha_mat.type() == CV_32FC1; const bool alpha_is_float = CV_MAT_DEPTH(alpha_mat.type()) == CV_32F;
RET_CHECK(alpha_is_float || alpha_mat.type() == CV_8UC1); RET_CHECK(alpha_is_float || CV_MAT_DEPTH(alpha_mat.type()) == CV_8U);
if (alpha_is_float) { if (alpha_is_float) {
MP_RETURN_IF_ERROR( MP_RETURN_IF_ERROR(
+2 -2
View File
@@ -12,10 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
licenses(["notice"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:private"]) package(default_visibility = ["//visibility:private"])
proto_library( proto_library(
+57 -135
View File
@@ -21,10 +21,12 @@ load(
) )
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test") load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto") load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
load("@org_tensorflow//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite")
load("//mediapipe/framework:more_selects.bzl", "more_selects")
licenses(["notice"]) licenses(["notice"])
package(default_visibility = ["//visibility:private"]) package(default_visibility = ["//visibility:public"])
exports_files( exports_files(
glob(["testdata/image_to_tensor/*"]), glob(["testdata/image_to_tensor/*"]),
@@ -34,6 +36,27 @@ exports_files(
], ],
) )
config_setting(
name = "disable_gpu",
define_values = {
"MEDIAPIPE_DISABLE_GPU": "1",
},
visibility = ["//visibility:public"],
)
more_selects.config_setting_negation(
name = "not_disable_gpu",
negate = ":disable_gpu",
)
selects.config_setting_group(
name = "platform_ios_with_gpu",
match_all = [
":not_disable_gpu",
"//mediapipe:ios",
],
)
selects.config_setting_group( selects.config_setting_group(
name = "compute_shader_unavailable", name = "compute_shader_unavailable",
match_any = [ match_any = [
@@ -44,9 +67,6 @@ selects.config_setting_group(
mediapipe_proto_library( mediapipe_proto_library(
name = "audio_to_tensor_calculator_proto", name = "audio_to_tensor_calculator_proto",
srcs = ["audio_to_tensor_calculator.proto"], srcs = ["audio_to_tensor_calculator.proto"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -56,17 +76,6 @@ mediapipe_proto_library(
cc_library( cc_library(
name = "audio_to_tensor_calculator", name = "audio_to_tensor_calculator",
srcs = ["audio_to_tensor_calculator.cc"], srcs = ["audio_to_tensor_calculator.cc"],
copts = select({
# b/215212850
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc",
],
"//conditions:default": [],
}),
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
":audio_to_tensor_calculator_cc_proto", ":audio_to_tensor_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -113,9 +122,6 @@ cc_test(
mediapipe_proto_library( mediapipe_proto_library(
name = "tensors_to_audio_calculator_proto", name = "tensors_to_audio_calculator_proto",
srcs = ["tensors_to_audio_calculator.proto"], srcs = ["tensors_to_audio_calculator.proto"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -125,9 +131,6 @@ mediapipe_proto_library(
cc_library( cc_library(
name = "tensors_to_audio_calculator", name = "tensors_to_audio_calculator",
srcs = ["tensors_to_audio_calculator.cc"], srcs = ["tensors_to_audio_calculator.cc"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
":tensors_to_audio_calculator_cc_proto", ":tensors_to_audio_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -164,9 +167,6 @@ cc_test(
mediapipe_proto_library( mediapipe_proto_library(
name = "feedback_tensors_calculator_proto", name = "feedback_tensors_calculator_proto",
srcs = ["feedback_tensors_calculator.proto"], srcs = ["feedback_tensors_calculator.proto"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -176,17 +176,6 @@ mediapipe_proto_library(
cc_library( cc_library(
name = "feedback_tensors_calculator", name = "feedback_tensors_calculator",
srcs = ["feedback_tensors_calculator.cc"], srcs = ["feedback_tensors_calculator.cc"],
copts = select({
# b/215212850
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc",
],
"//conditions:default": [],
}),
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
":feedback_tensors_calculator_cc_proto", ":feedback_tensors_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -216,9 +205,6 @@ cc_test(
mediapipe_proto_library( mediapipe_proto_library(
name = "bert_preprocessor_calculator_proto", name = "bert_preprocessor_calculator_proto",
srcs = ["bert_preprocessor_calculator.proto"], srcs = ["bert_preprocessor_calculator.proto"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -228,9 +214,6 @@ mediapipe_proto_library(
cc_library( cc_library(
name = "bert_preprocessor_calculator", name = "bert_preprocessor_calculator",
srcs = ["bert_preprocessor_calculator.cc"], srcs = ["bert_preprocessor_calculator.cc"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
":bert_preprocessor_calculator_cc_proto", ":bert_preprocessor_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -274,9 +257,6 @@ cc_test(
mediapipe_proto_library( mediapipe_proto_library(
name = "regex_preprocessor_calculator_proto", name = "regex_preprocessor_calculator_proto",
srcs = ["regex_preprocessor_calculator.proto"], srcs = ["regex_preprocessor_calculator.proto"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -286,9 +266,6 @@ mediapipe_proto_library(
cc_library( cc_library(
name = "regex_preprocessor_calculator", name = "regex_preprocessor_calculator",
srcs = ["regex_preprocessor_calculator.cc"], srcs = ["regex_preprocessor_calculator.cc"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
":regex_preprocessor_calculator_cc_proto", ":regex_preprocessor_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -330,9 +307,6 @@ cc_test(
cc_library( cc_library(
name = "text_to_tensor_calculator", name = "text_to_tensor_calculator",
srcs = ["text_to_tensor_calculator.cc"], srcs = ["text_to_tensor_calculator.cc"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
"//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -366,9 +340,6 @@ cc_test(
cc_library( cc_library(
name = "universal_sentence_encoder_preprocessor_calculator", name = "universal_sentence_encoder_preprocessor_calculator",
srcs = ["universal_sentence_encoder_preprocessor_calculator.cc"], srcs = ["universal_sentence_encoder_preprocessor_calculator.cc"],
visibility = [
"//mediapipe/framework:mediapipe_internal",
],
deps = [ deps = [
"//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -408,7 +379,6 @@ cc_test(
mediapipe_proto_library( mediapipe_proto_library(
name = "inference_calculator_proto", name = "inference_calculator_proto",
srcs = ["inference_calculator.proto"], srcs = ["inference_calculator.proto"],
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -423,19 +393,15 @@ mediapipe_proto_library(
# size concerns), depend on those implementations directly, and do not depend on # size concerns), depend on those implementations directly, and do not depend on
# :inference_calculator. # :inference_calculator.
# In all cases, use "InferenceCalulator" in your graphs. # In all cases, use "InferenceCalulator" in your graphs.
cc_library( cc_library_with_tflite(
name = "inference_calculator_interface", name = "inference_calculator_interface",
srcs = ["inference_calculator.cc"], srcs = ["inference_calculator.cc"],
hdrs = ["inference_calculator.h"], hdrs = ["inference_calculator.h"],
copts = select({ tflite_deps = [
# TODO: fix tensor.h not to require this, if possible "//mediapipe/util/tflite:tflite_model_loader",
"//mediapipe:apple": [ "@org_tensorflow//tensorflow/lite/core/shims:framework_stable",
"-x objective-c++", "@org_tensorflow//tensorflow/lite/core/shims:builtin_ops",
"-fobjc-arc", # enable reference-counting ],
],
"//conditions:default": [],
}),
visibility = ["//visibility:public"],
deps = [ deps = [
":inference_calculator_cc_proto", ":inference_calculator_cc_proto",
":inference_calculator_options_lib", ":inference_calculator_options_lib",
@@ -446,12 +412,9 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
"//mediapipe/framework/tool:subgraph_expansion", "//mediapipe/framework/tool:subgraph_expansion",
"//mediapipe/util/tflite:tflite_model_loader",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/core/api:op_resolver", "@org_tensorflow//tensorflow/lite/core/api:op_resolver",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -460,7 +423,6 @@ cc_library(
name = "inference_calculator_gl", name = "inference_calculator_gl",
srcs = ["inference_calculator_gl.cc"], srcs = ["inference_calculator_gl.cc"],
tags = ["nomac"], # config problem with cpuinfo via TF tags = ["nomac"], # config problem with cpuinfo via TF
visibility = ["//visibility:public"],
deps = [ deps = [
":inference_calculator_cc_proto", ":inference_calculator_cc_proto",
":inference_calculator_interface", ":inference_calculator_interface",
@@ -478,7 +440,6 @@ cc_library(
name = "inference_calculator_gl_advanced", name = "inference_calculator_gl_advanced",
srcs = ["inference_calculator_gl_advanced.cc"], srcs = ["inference_calculator_gl_advanced.cc"],
tags = ["nomac"], tags = ["nomac"],
visibility = ["//visibility:public"],
deps = [ deps = [
":inference_calculator_interface", ":inference_calculator_interface",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
@@ -509,9 +470,9 @@ cc_library(
"-framework MetalKit", "-framework MetalKit",
], ],
tags = ["ios"], tags = ["ios"],
visibility = ["//visibility:public"],
deps = [ deps = [
"inference_calculator_interface", "inference_calculator_interface",
"//mediapipe/framework/formats:tensor",
"//mediapipe/gpu:MPPMetalHelper", "//mediapipe/gpu:MPPMetalHelper",
"//mediapipe/gpu:MPPMetalUtil", "//mediapipe/gpu:MPPMetalUtil",
"//mediapipe/gpu:gpu_buffer", "//mediapipe/gpu:gpu_buffer",
@@ -530,15 +491,6 @@ cc_library(
cc_library( cc_library(
name = "inference_runner", name = "inference_runner",
hdrs = ["inference_runner.h"], hdrs = ["inference_runner.h"],
copts = select({
# TODO: fix tensor.h not to require this, if possible
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc", # enable reference-counting
],
"//conditions:default": [],
}),
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_context",
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
@@ -546,31 +498,33 @@ cc_library(
], ],
) )
cc_library( cc_library_with_tflite(
name = "tflite_delegate_ptr",
hdrs = ["tflite_delegate_ptr.h"],
tflite_deps = [
"@org_tensorflow//tensorflow/lite/core/shims:c_api_types",
],
)
cc_library_with_tflite(
name = "inference_interpreter_delegate_runner", name = "inference_interpreter_delegate_runner",
srcs = ["inference_interpreter_delegate_runner.cc"], srcs = ["inference_interpreter_delegate_runner.cc"],
hdrs = ["inference_interpreter_delegate_runner.h"], hdrs = ["inference_interpreter_delegate_runner.h"],
copts = select({ tflite_deps = [
# TODO: fix tensor.h not to require this, if possible ":tflite_delegate_ptr",
"//mediapipe:apple": [ "//mediapipe/util/tflite:tflite_model_loader",
"-x objective-c++", "@org_tensorflow//tensorflow/lite/core/shims:c_api_types",
"-fobjc-arc", # enable reference-counting "@org_tensorflow//tensorflow/lite/core/shims:framework_stable",
], ],
"//conditions:default": [],
}),
visibility = ["//visibility:public"],
deps = [ deps = [
":inference_runner", ":inference_runner",
"//mediapipe/framework:mediapipe_profiling", "//mediapipe/framework:mediapipe_profiling",
"//mediapipe/framework/api2:packet", "//mediapipe/framework/api2:packet",
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/util/tflite:tflite_model_loader",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor", "@com_google_absl//absl/status:statusor",
"@org_tensorflow//tensorflow/lite:framework_stable",
"@org_tensorflow//tensorflow/lite:string_util", "@org_tensorflow//tensorflow/lite:string_util",
"@org_tensorflow//tensorflow/lite/c:c_api_types",
"@org_tensorflow//tensorflow/lite/core/api:op_resolver", "@org_tensorflow//tensorflow/lite/core/api:op_resolver",
], ],
) )
@@ -580,15 +534,6 @@ cc_library(
srcs = [ srcs = [
"inference_calculator_cpu.cc", "inference_calculator_cpu.cc",
], ],
copts = select({
# TODO: fix tensor.h not to require this, if possible
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc", # enable reference-counting
],
"//conditions:default": [],
}),
visibility = ["//visibility:public"],
deps = [ deps = [
":inference_calculator_interface", ":inference_calculator_interface",
":inference_calculator_utils", ":inference_calculator_utils",
@@ -597,9 +542,9 @@ cc_library(
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor", "@com_google_absl//absl/status:statusor",
"@org_tensorflow//tensorflow/lite/core/shims:c_api_types",
"@org_tensorflow//tensorflow/lite/core/shims:framework_stable",
"@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate", "@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
"@org_tensorflow//tensorflow/lite:framework_stable",
"@org_tensorflow//tensorflow/lite/c:c_api_types",
] + select({ ] + select({
"//conditions:default": [], "//conditions:default": [],
"//mediapipe:android": ["@org_tensorflow//tensorflow/lite/delegates/nnapi:nnapi_delegate"], "//mediapipe:android": ["@org_tensorflow//tensorflow/lite/delegates/nnapi:nnapi_delegate"],
@@ -627,15 +572,6 @@ cc_library(
srcs = [ srcs = [
"inference_calculator_xnnpack.cc", "inference_calculator_xnnpack.cc",
], ],
copts = select({
# TODO: fix tensor.h not to require this, if possible
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc", # enable reference-counting
],
"//conditions:default": [],
}),
visibility = ["//visibility:public"],
deps = [ deps = [
":inference_calculator_interface", ":inference_calculator_interface",
":inference_calculator_utils", ":inference_calculator_utils",
@@ -651,7 +587,6 @@ cc_library(
cc_library( cc_library(
name = "inference_calculator_gl_if_compute_shader_available", name = "inference_calculator_gl_if_compute_shader_available",
visibility = ["//visibility:public"],
deps = selects.with_or({ deps = selects.with_or({
":compute_shader_unavailable": [], ":compute_shader_unavailable": [],
"//conditions:default": [ "//conditions:default": [
@@ -667,13 +602,12 @@ cc_library(
# inference_calculator_interface. # inference_calculator_interface.
cc_library( cc_library(
name = "inference_calculator", name = "inference_calculator",
visibility = ["//visibility:public"],
deps = [ deps = [
":inference_calculator_interface", ":inference_calculator_interface",
":inference_calculator_cpu", ":inference_calculator_cpu",
] + select({ ] + select({
"//conditions:default": [":inference_calculator_gl_if_compute_shader_available"], "//conditions:default": [":inference_calculator_gl_if_compute_shader_available"],
"//mediapipe:ios": [":inference_calculator_metal"], ":platform_ios_with_gpu": [":inference_calculator_metal"],
}), }),
alwayslink = 1, alwayslink = 1,
) )
@@ -681,7 +615,6 @@ cc_library(
mediapipe_proto_library( mediapipe_proto_library(
name = "tensor_converter_calculator_proto", name = "tensor_converter_calculator_proto",
srcs = ["tensor_converter_calculator.proto"], srcs = ["tensor_converter_calculator.proto"],
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -706,7 +639,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":tensor_converter_calculator_cc_proto", ":tensor_converter_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -725,6 +657,7 @@ cc_library(
cc_library( cc_library(
name = "tensor_converter_calculator_gpu_deps", name = "tensor_converter_calculator_gpu_deps",
visibility = ["//visibility:private"],
deps = select({ deps = select({
"//mediapipe:android": [ "//mediapipe:android": [
"//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_calculator_helper",
@@ -769,7 +702,6 @@ cc_test(
mediapipe_proto_library( mediapipe_proto_library(
name = "tensors_to_detections_calculator_proto", name = "tensors_to_detections_calculator_proto",
srcs = ["tensors_to_detections_calculator.proto"], srcs = ["tensors_to_detections_calculator.proto"],
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -794,7 +726,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":tensors_to_detections_calculator_cc_proto", ":tensors_to_detections_calculator_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:detection_cc_proto",
@@ -817,6 +748,7 @@ cc_library(
cc_library( cc_library(
name = "tensors_to_detections_calculator_gpu_deps", name = "tensors_to_detections_calculator_gpu_deps",
visibility = ["//visibility:private"],
deps = select({ deps = select({
"//mediapipe:ios": [ "//mediapipe:ios": [
"//mediapipe/gpu:MPPMetalUtil", "//mediapipe/gpu:MPPMetalUtil",
@@ -832,7 +764,6 @@ cc_library(
mediapipe_proto_library( mediapipe_proto_library(
name = "tensors_to_landmarks_calculator_proto", name = "tensors_to_landmarks_calculator_proto",
srcs = ["tensors_to_landmarks_calculator.proto"], srcs = ["tensors_to_landmarks_calculator.proto"],
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -849,7 +780,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":tensors_to_landmarks_calculator_cc_proto", ":tensors_to_landmarks_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -864,7 +794,6 @@ cc_library(
mediapipe_proto_library( mediapipe_proto_library(
name = "landmarks_to_tensor_calculator_proto", name = "landmarks_to_tensor_calculator_proto",
srcs = ["landmarks_to_tensor_calculator.proto"], srcs = ["landmarks_to_tensor_calculator.proto"],
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -882,7 +811,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":landmarks_to_tensor_calculator_cc_proto", ":landmarks_to_tensor_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -915,7 +843,6 @@ cc_test(
mediapipe_proto_library( mediapipe_proto_library(
name = "tensors_to_floats_calculator_proto", name = "tensors_to_floats_calculator_proto",
srcs = ["tensors_to_floats_calculator.proto"], srcs = ["tensors_to_floats_calculator.proto"],
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -932,7 +859,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":tensors_to_floats_calculator_cc_proto", ":tensors_to_floats_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -970,7 +896,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":tensors_to_classification_calculator_cc_proto", ":tensors_to_classification_calculator_cc_proto",
"@com_google_absl//absl/container:node_hash_map", "@com_google_absl//absl/container:node_hash_map",
@@ -1001,7 +926,6 @@ cc_library(
mediapipe_proto_library( mediapipe_proto_library(
name = "tensors_to_classification_calculator_proto", name = "tensors_to_classification_calculator_proto",
srcs = ["tensors_to_classification_calculator.proto"], srcs = ["tensors_to_classification_calculator.proto"],
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -1039,7 +963,6 @@ cc_library(
"//conditions:default": [], "//conditions:default": [],
}), }),
features = ["-layering_check"], # allow depending on image_to_tensor_calculator_gpu_deps features = ["-layering_check"], # allow depending on image_to_tensor_calculator_gpu_deps
visibility = ["//visibility:public"],
deps = [ deps = [
":image_to_tensor_calculator_cc_proto", ":image_to_tensor_calculator_cc_proto",
":image_to_tensor_converter", ":image_to_tensor_converter",
@@ -1068,6 +991,7 @@ cc_library(
cc_library( cc_library(
name = "image_to_tensor_calculator_gpu_deps", name = "image_to_tensor_calculator_gpu_deps",
visibility = ["//visibility:private"],
deps = selects.with_or({ deps = selects.with_or({
"//mediapipe:android": [ "//mediapipe:android": [
":image_to_tensor_converter_gl_buffer", ":image_to_tensor_converter_gl_buffer",
@@ -1091,7 +1015,6 @@ cc_library(
mediapipe_proto_library( mediapipe_proto_library(
name = "image_to_tensor_calculator_proto", name = "image_to_tensor_calculator_proto",
srcs = ["image_to_tensor_calculator.proto"], srcs = ["image_to_tensor_calculator.proto"],
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -1141,6 +1064,7 @@ cc_test(
"@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
], ],
) )
@@ -1154,7 +1078,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":image_to_tensor_utils", ":image_to_tensor_utils",
"//mediapipe/framework/formats:image", "//mediapipe/framework/formats:image",
@@ -1174,7 +1097,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":image_to_tensor_converter", ":image_to_tensor_converter",
":image_to_tensor_utils", ":image_to_tensor_utils",
@@ -1194,6 +1116,7 @@ cc_library(
name = "image_to_tensor_converter_gl_buffer", name = "image_to_tensor_converter_gl_buffer",
srcs = ["image_to_tensor_converter_gl_buffer.cc"], srcs = ["image_to_tensor_converter_gl_buffer.cc"],
hdrs = ["image_to_tensor_converter_gl_buffer.h"], hdrs = ["image_to_tensor_converter_gl_buffer.h"],
visibility = ["//visibility:private"],
deps = ["//mediapipe/framework:port"] + selects.with_or({ deps = ["//mediapipe/framework:port"] + selects.with_or({
"//mediapipe:apple": [], "//mediapipe:apple": [],
"//conditions:default": [ "//conditions:default": [
@@ -1227,6 +1150,7 @@ cc_library(
name = "image_to_tensor_converter_gl_texture", name = "image_to_tensor_converter_gl_texture",
srcs = ["image_to_tensor_converter_gl_texture.cc"], srcs = ["image_to_tensor_converter_gl_texture.cc"],
hdrs = ["image_to_tensor_converter_gl_texture.h"], hdrs = ["image_to_tensor_converter_gl_texture.h"],
visibility = ["//visibility:private"],
deps = ["//mediapipe/framework:port"] + select({ deps = ["//mediapipe/framework:port"] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [ "//conditions:default": [
@@ -1251,6 +1175,7 @@ cc_library(
name = "image_to_tensor_converter_gl_utils", name = "image_to_tensor_converter_gl_utils",
srcs = ["image_to_tensor_converter_gl_utils.cc"], srcs = ["image_to_tensor_converter_gl_utils.cc"],
hdrs = ["image_to_tensor_converter_gl_utils.h"], hdrs = ["image_to_tensor_converter_gl_utils.h"],
visibility = ["//visibility:private"],
deps = ["//mediapipe/framework:port"] + select({ deps = ["//mediapipe/framework:port"] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [ "//conditions:default": [
@@ -1280,6 +1205,7 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:private"],
deps = ["//mediapipe/framework:port"] + select({ deps = ["//mediapipe/framework:port"] + select({
"//mediapipe:apple": [ "//mediapipe:apple": [
":image_to_tensor_converter", ":image_to_tensor_converter",
@@ -1311,7 +1237,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":image_to_tensor_calculator_cc_proto", ":image_to_tensor_calculator_cc_proto",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
@@ -1354,7 +1279,6 @@ selects.config_setting_group(
mediapipe_proto_library( mediapipe_proto_library(
name = "tensors_to_segmentation_calculator_proto", name = "tensors_to_segmentation_calculator_proto",
srcs = ["tensors_to_segmentation_calculator.proto"], srcs = ["tensors_to_segmentation_calculator.proto"],
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
@@ -1372,7 +1296,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
":tensors_to_segmentation_calculator_cc_proto", ":tensors_to_segmentation_calculator_cc_proto",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
@@ -1430,7 +1353,6 @@ cc_library(
], ],
"//conditions:default": [], "//conditions:default": [],
}), }),
visibility = ["//visibility:public"],
deps = [ deps = [
"//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
@@ -203,6 +203,7 @@ class AudioToTensorCalculator : public Node {
std::unique_ptr<audio_dsp::QResampler<float>> resampler_; std::unique_ptr<audio_dsp::QResampler<float>> resampler_;
Matrix sample_buffer_; Matrix sample_buffer_;
int processed_buffer_cols_ = 0; int processed_buffer_cols_ = 0;
double gain_ = 1.0;
// The internal state of the FFT library. // The internal state of the FFT library.
PFFFT_Setup* fft_state_ = nullptr; PFFFT_Setup* fft_state_ = nullptr;
@@ -278,7 +279,9 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
padding_samples_after_ = options.padding_samples_after(); padding_samples_after_ = options.padding_samples_after();
dft_tensor_format_ = options.dft_tensor_format(); dft_tensor_format_ = options.dft_tensor_format();
flush_mode_ = options.flush_mode(); flush_mode_ = options.flush_mode();
if (options.has_volume_gain_db()) {
gain_ = pow(10, options.volume_gain_db() / 20.0);
}
RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^ RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
!kAudioIn(cc).Header().IsEmpty()) !kAudioIn(cc).Header().IsEmpty())
<< "Must either specify the time series header of the \"AUDIO\" stream " << "Must either specify the time series header of the \"AUDIO\" stream "
@@ -344,6 +347,10 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
const Matrix& input = channels_match ? input_frame const Matrix& input = channels_match ? input_frame
// Mono mixdown. // Mono mixdown.
: input_frame.colwise().mean(); : input_frame.colwise().mean();
if (gain_ != 1.0) {
return stream_mode_ ? ProcessStreamingData(cc, input * gain_)
: ProcessNonStreamingData(cc, input * gain_);
}
return stream_mode_ ? ProcessStreamingData(cc, input) return stream_mode_ ? ProcessStreamingData(cc, input)
: ProcessNonStreamingData(cc, input); : ProcessNonStreamingData(cc, input);
} }
@@ -81,4 +81,8 @@ message AudioToTensorCalculatorOptions {
WITH_DC_AND_NYQUIST = 3; WITH_DC_AND_NYQUIST = 3;
} }
optional DftTensorFormat dft_tensor_format = 11 [default = WITH_NYQUIST]; optional DftTensorFormat dft_tensor_format = 11 [default = WITH_NYQUIST];
// The volume gain, measured in dB.
// Scale the input audio amplitude by 10^(volume_gain_db/20).
optional double volume_gain_db = 12;
} }
@@ -195,8 +195,9 @@ class ImageToTensorCalculator : public Node {
#endif // MEDIAPIPE_DISABLE_GPU #endif // MEDIAPIPE_DISABLE_GPU
RotatedRect roi = GetRoi(image->width(), image->height(), norm_rect); RotatedRect roi = GetRoi(image->width(), image->height(), norm_rect);
ASSIGN_OR_RETURN(auto padding, PadRoi(options_.output_tensor_width(), const int tensor_width = params_.output_width.value_or(image->width());
options_.output_tensor_height(), const int tensor_height = params_.output_height.value_or(image->height());
ASSIGN_OR_RETURN(auto padding, PadRoi(tensor_width, tensor_height,
options_.keep_aspect_ratio(), &roi)); options_.keep_aspect_ratio(), &roi));
if (kOutLetterboxPadding(cc).IsConnected()) { if (kOutLetterboxPadding(cc).IsConnected()) {
kOutLetterboxPadding(cc).Send(padding); kOutLetterboxPadding(cc).Send(padding);
@@ -214,9 +215,8 @@ class ImageToTensorCalculator : public Node {
Tensor::ElementType output_tensor_type = Tensor::ElementType output_tensor_type =
GetOutputTensorType(image->UsesGpu(), params_); GetOutputTensorType(image->UsesGpu(), params_);
Tensor tensor(output_tensor_type, Tensor tensor(output_tensor_type, {1, tensor_height, tensor_width,
{1, params_.output_height, params_.output_width, GetNumOutputChannels(*image)});
GetNumOutputChannels(*image)});
MP_RETURN_IF_ERROR((image->UsesGpu() ? gpu_converter_ : cpu_converter_) MP_RETURN_IF_ERROR((image->UsesGpu() ? gpu_converter_ : cpu_converter_)
->Convert(*image, roi, params_.range_min, ->Convert(*image, roi, params_.range_min,
params_.range_max, params_.range_max,

Some files were not shown because too many files have changed in this diff Show More