Compare commits

..
Author SHA1 Message Date
Sebastian SchmidtandGitHub 7d05709f62 Revert "Revert "Add Face Landmarks Connection constants for the Python API"" 2023-04-13 14:14:12 -06:00
Sebastian SchmidtandGitHub f712eed6c3 Merge pull request #4286 from google/revert-4279-face-landmarker-python-add-connection-constants
Revert "Add Face Landmarks Connection constants for the Python API"
2023-04-13 14:14:02 -06:00
yichunkuoandGitHub 7cbf0033b0 Revert "Add Face Landmarks Connection constants for the Python API" 2023-04-14 04:12:33 +08:00
MediaPipe TeamandCopybara-Service a59b29ea24 Add jni functions to get image size from a list of image
PiperOrigin-RevId: 524080111
2023-04-13 13:05:42 -07:00
MediaPipe TeamandCopybara-Service ad169b9123 Add support for more standard scaling options in GlSurfaceViewRenderer.
PiperOrigin-RevId: 524062033
2023-04-13 11:52:45 -07:00
MediaPipe TeamandCopybara-Service c63b30dff5 Internal change
PiperOrigin-RevId: 524059494
2023-04-13 11:45:04 -07:00
MediaPipe TeamandCopybara-Service 055accece8 Internal Change
PiperOrigin-RevId: 524035898
2023-04-13 11:43:20 -07:00
yichunkuoandGitHub 78499b0bd0 Merge pull request #4279 from kinaryml/face-landmarker-python-add-connection-constants
Add Face Landmarks Connection constants for the Python API
2023-04-14 02:35:22 +08:00
Fergus HendersonandCopybara-Service c392ed2c09 Internal change
PiperOrigin-RevId: 524012087
2023-04-13 08:49:36 -07:00
MediaPipe TeamandCopybara-Service d095829c7c Internal cleanup.
PiperOrigin-RevId: 524005450
2023-04-13 08:19:29 -07:00
MediaPipe TeamandCopybara-Service 6af4b433f4 Internal change
PiperOrigin-RevId: 523965337
2023-04-13 04:36:16 -07:00
MediaPipe TeamandCopybara-Service 3fdd7f3618 Internal change
PiperOrigin-RevId: 523930101
2023-04-13 01:28:41 -07:00
kinaryml 4d5f081232 Reformatted face_landmarker.py 2023-04-13 01:18:37 -07:00
Kinar RandGitHub dc56eb2bb1 Update face_landmarker.py 2023-04-13 13:40:49 +05:30
Kinar RandGitHub 5185489c21 Update face_landmarker.py 2023-04-13 13:30:00 +05:30
MediaPipe TeamandCopybara-Service c8b9cec7ef Internal change
PiperOrigin-RevId: 523916108
2023-04-13 00:16:17 -07:00
MediaPipe TeamandCopybara-Service d96b6e7ed9 Make confidence and category masks both optional for GPU multi-class segmentation.
PiperOrigin-RevId: 523838907
2023-04-12 16:41:14 -07:00
Sebastian SchmidtandCopybara-Service f23dd69db1 Fix yet another Windows compile error
PiperOrigin-RevId: 523835509
2023-04-12 16:26:27 -07:00
Sebastian SchmidtandCopybara-Service d0e8a9e09b Internal change
PiperOrigin-RevId: 523827005
2023-04-12 15:49:58 -07:00
MediaPipe TeamandCopybara-Service 0179f0c456 Update TensorsToFaceLandmarksGraph to support face mesh v2 model.
PiperOrigin-RevId: 523814749
2023-04-12 15:03:26 -07:00
Sebastian SchmidtandCopybara-Service 468d10e947 Update WASM files for Alpha 10
PiperOrigin-RevId: 523810757
2023-04-12 14:47:22 -07:00
Copybara-Service dd62b0831a Merge pull request #4254 from priankakariatyml:ios-gesture-recognizer
PiperOrigin-RevId: 523808915
2023-04-12 14:40:31 -07:00
MediaPipe TeamandCopybara-Service a71c697d90 Internal change
PiperOrigin-RevId: 523797476
2023-04-12 13:58:11 -07:00
MediaPipe TeamandCopybara-Service 27c38f00ec Add pose landmarker C++ API.
PiperOrigin-RevId: 523795237
2023-04-12 13:51:54 -07:00
MediaPipe TeamandCopybara-Service c7aecb42ff Internal change
PiperOrigin-RevId: 523785788
2023-04-12 13:16:33 -07:00
kinaryml fec96ee679 Added some face landmarks constants 2023-04-12 13:03:46 -07:00
MediaPipe TeamandCopybara-Service 9a10375de6 internal change
PiperOrigin-RevId: 523773255
2023-04-12 12:27:59 -07:00
MediaPipe TeamandCopybara-Service ca0da8d26f Fix wrong function name of ConvertToDetection
PiperOrigin-RevId: 523760961
2023-04-12 11:42:48 -07:00
MediaPipe TeamandCopybara-Service 776dceb588 Build category mask without resizing tensors to image size.
PiperOrigin-RevId: 523754567
2023-04-12 11:22:30 -07:00
MediaPipe TeamandCopybara-Service 049ba8bbca Internal change
PiperOrigin-RevId: 523751152
2023-04-12 11:10:43 -07:00
Sebastian SchmidtandCopybara-Service f9a2d0995d Fix Windows compilation error
PiperOrigin-RevId: 523739720
2023-04-12 10:34:21 -07:00
MediaPipe TeamandCopybara-Service efab342a52 Internal change
PiperOrigin-RevId: 523733431
2023-04-12 10:16:15 -07:00
MediaPipe TeamandCopybara-Service 5bcdea2952 Internal MediaPipe Tasks change.
PiperOrigin-RevId: 523729183
2023-04-12 09:59:21 -07:00
MediaPipe TeamandCopybara-Service f41a912250 Internal change
PiperOrigin-RevId: 523703537
2023-04-12 08:14:13 -07:00
MediaPipe TeamandCopybara-Service 2eefc862c2 Internal change
PiperOrigin-RevId: 523632081
2023-04-12 01:25:23 -07:00
MediaPipe TeamandCopybara-Service a7c6910625 Internal change
PiperOrigin-RevId: 523616811
2023-04-11 23:52:14 -07:00
MediaPipe TeamandCopybara-Service 52e934a539 Internal change
PiperOrigin-RevId: 523611358
2023-04-11 23:13:45 -07:00
MediaPipe TeamandCopybara-Service f923f6bcda Internal change
PiperOrigin-RevId: 523589395
2023-04-11 21:03:38 -07:00
MediaPipe TeamandCopybara-Service c9aa24b0e7 Support proto3 node_option in api2 graph_builder
PiperOrigin-RevId: 523587324
2023-04-11 20:49:35 -07:00
MediaPipe TeamandCopybara-Service b6a19ea9e8 Add shaders that support better landscape rendering with GlSurfaceViewRenderer.
PiperOrigin-RevId: 523579769
2023-04-11 20:10:40 -07:00
Sebastian SchmidtandCopybara-Service 1b947df0c2 Allow relative URLs for WASM loading
PiperOrigin-RevId: 523572715
2023-04-11 19:25:03 -07:00
MediaPipe TeamandCopybara-Service e2b13523f1 F16 texture support in GPU multiclass TensorsToSegmentation
PiperOrigin-RevId: 523512145
2023-04-11 14:58:57 -07:00
Sebastian SchmidtandCopybara-Service 738aea7a06 Update WASM files for alpha 9
PiperOrigin-RevId: 523458140
2023-04-11 11:33:14 -07:00
Sebastian SchmidtandCopybara-Service 221d545080 Create Language Detection Web API
PiperOrigin-RevId: 523445524
2023-04-11 10:47:29 -07:00
Sebastian SchmidtandCopybara-Service b66b0e0c72 Internal change
PiperOrigin-RevId: 523429555
2023-04-11 09:56:46 -07:00
MediaPipe TeamandCopybara-Service a448790300 Internal change
PiperOrigin-RevId: 523351901
2023-04-11 03:23:24 -07:00
MediaPipe TeamandCopybara-Service 9793654364 Update Halide build rules for MediaPipe to use Halide v15.0.1
PiperOrigin-RevId: 523351862
2023-04-11 03:18:54 -07:00
MediaPipe TeamandCopybara-Service de7560c825 Internal change
PiperOrigin-RevId: 523339435
2023-04-11 02:11:41 -07:00
MediaPipe TeamandCopybara-Service b8ebbbea0b Internal change
PiperOrigin-RevId: 523324063
2023-04-11 00:46:49 -07:00
MediaPipe TeamandCopybara-Service a4172cb03f Internal change
PiperOrigin-RevId: 523306493
2023-04-10 22:50:58 -07:00
MediaPipe TeamandCopybara-Service 3bb411e99d Special treatment for 1-class segmentation category mask output on GPU.
PiperOrigin-RevId: 523271622
2023-04-10 19:13:38 -07:00
Sebastian SchmidtandCopybara-Service 87ec846ed6 Enable TextClassifier and TextEmbedder on Windows Python
PiperOrigin-RevId: 523233995
2023-04-10 15:57:19 -07:00
MediaPipe TeamandCopybara-Service c1f17138cf Upgrades and fixes for image segmentation category mask on GPU
PiperOrigin-RevId: 523204584
2023-04-10 13:58:20 -07:00
MediaPipe TeamandCopybara-Service 02fed0b7d1 Cover the existing graph expansion behavior in regard to executors with unit tests.
PiperOrigin-RevId: 523192292
2023-04-10 13:07:51 -07:00
Sebastian SchmidtandCopybara-Service e5f28bc136 Make AudioTools compile when build from python:framework_bindings
PiperOrigin-RevId: 523158401
2023-04-10 11:00:15 -07:00
Copybara-Service b685f53c6b Merge pull request #4263 from kinaryml:face-stylizer-python-tests
PiperOrigin-RevId: 523152290
2023-04-10 10:39:50 -07:00
MediaPipe TeamandCopybara-Service c036b9f408 Adds a LanguageDetector Java API.
PiperOrigin-RevId: 522895455
2023-04-08 22:31:28 -07:00
MediaPipe TeamandCopybara-Service 4f77504af6 Internal change
PiperOrigin-RevId: 522787446
2023-04-08 03:01:54 -07:00
kinaryml 18d5beb598 Updated FaceStylizer API to align with the new Base Vision Task API changes 2023-04-08 01:38:48 -07:00
MediaPipe TeamandCopybara-Service 7b067a1fda Add face landmarks connection for java API.
PiperOrigin-RevId: 522728488
2023-04-07 18:27:39 -07:00
Sebastian SchmidtandCopybara-Service e19b49ba13 Fix indent
PiperOrigin-RevId: 522715076
2023-04-07 16:59:03 -07:00
Sebastian SchmidtandCopybara-Service 2efcf30eea Add FaceLandmarksConnections to Web API
PiperOrigin-RevId: 522713874
2023-04-07 16:53:57 -07:00
Sebastian SchmidtandCopybara-Service 938b501d15 Don't use OffscreenCanvas on Safari
PiperOrigin-RevId: 522689566
2023-04-07 14:51:18 -07:00
Jiuqiang TangandCopybara-Service 8cedb82df1 Expose face stylizer and interactive segmenter to be public mediapipe python API.
PiperOrigin-RevId: 522654239
2023-04-07 12:12:49 -07:00
MediaPipe TeamandCopybara-Service a1ce19f68e Internal change
PiperOrigin-RevId: 522631851
2023-04-07 10:47:36 -07:00
MediaPipe TeamandCopybara-Service e3185e3df0 Internal change
PiperOrigin-RevId: 522614549
2023-04-07 09:30:36 -07:00
Jiuqiang TangandCopybara-Service bca0a92c2e Add FaceToRectCalculator.
PiperOrigin-RevId: 522595752
2023-04-07 07:44:51 -07:00
MediaPipe TeamandCopybara-Service b0d3595291 Internal change
PiperOrigin-RevId: 522541374
2023-04-07 00:30:49 -07:00
MediaPipe TeamandCopybara-Service c6b3090d0e Internal change
PiperOrigin-RevId: 522534050
2023-04-06 23:31:15 -07:00
MediaPipe TeamandCopybara-Service 6c523cd21f Internal change
PiperOrigin-RevId: 522529789
2023-04-06 22:56:06 -07:00
MediaPipe TeamandCopybara-Service bae14a83b2 Internal change
PiperOrigin-RevId: 522524565
2023-04-06 22:13:38 -07:00
MediaPipe TeamandCopybara-Service 4e5f20f212 Internal change
PiperOrigin-RevId: 522494903
2023-04-06 19:02:16 -07:00
Copybara-Service 6495f8624b Merge pull request #4241 from kinaryml:interactive-segmenter-python
PiperOrigin-RevId: 522470912
2023-04-06 16:47:46 -07:00
MediaPipe TeamandCopybara-Service 38f838513a Enable defining and using internal executor within a subgraph
PiperOrigin-RevId: 522449982
2023-04-06 15:18:56 -07:00
Prianka Liz Kariat fe5ca09030 Added MPPGestureRecognizerOptions.m 2023-04-07 01:01:39 +05:30
Prianka Liz Kariat 49c614280f Added MPPGestureRecognizerOptions 2023-04-07 00:58:53 +05:30
Prianka Liz Kariat 6f5b12d056 Added MPPGestureRecognizerResult 2023-04-07 00:58:26 +05:30
Prianka Liz Kariat 1992dfdf2c Added MPPClassifierOptions 2023-04-07 00:57:52 +05:30
Prianka Liz Kariat 9ce300c711 Added MPPClassifierOptions 2023-04-07 00:57:33 +05:30
Prianka Liz Kariat 5272980c2e Added MPPLandmarkHelpers 2023-04-07 00:57:19 +05:30
Prianka Liz Kariat cfa261b34f Added MPPLandmark 2023-04-07 00:57:01 +05:30
Copybara-Service 81a405af1b Merge pull request #4253 from priankakariatyml:object-detector-sources
PiperOrigin-RevId: 522401491
2023-04-06 12:15:34 -07:00
MediaPipe TeamandCopybara-Service 489d684699 Internal change
PiperOrigin-RevId: 522398388
2023-04-06 12:04:03 -07:00
Sebastian SchmidtandCopybara-Service 3588be3342 Update WASM files for Alpha 8
PiperOrigin-RevId: 522376911
2023-04-06 10:49:32 -07:00
Copybara-Service 37334098b0 Merge pull request #4248 from Neilblaze:docs-fix_breaks
PiperOrigin-RevId: 522355500
2023-04-06 09:29:52 -07:00
Prianka Liz Kariat 30341024de Added MPPObjectDetector 2023-04-06 21:40:52 +05:30
Prianka Liz Kariat b01b3b84c4 Added MPPObjectDetectionResultHelpers 2023-04-06 21:40:09 +05:30
Prianka Liz Kariat 79c8cfb730 Added MPPObjectDetectorOptionsHelpers 2023-04-06 21:39:56 +05:30
MediaPipe TeamandCopybara-Service 7d8d3ab196 Add EDGETPU_NNAPI delegate option in MediaPipe tasks API
PiperOrigin-RevId: 522344828
2023-04-06 08:42:15 -07:00
MediaPipe TeamandCopybara-Service 97bd9c2157 Internal change
PiperOrigin-RevId: 522307800
2023-04-06 05:01:29 -07:00
MediaPipe TeamandCopybara-Service 2e256bebb5 Internal change
PiperOrigin-RevId: 522292310
2023-04-06 03:20:04 -07:00
MediaPipe TeamandCopybara-Service e894ae9cf4 Internal change
PiperOrigin-RevId: 522291640
2023-04-06 03:15:24 -07:00
MediaPipe TeamandCopybara-Service 289f51651f Make GraphTextureFrame constructor public.
PiperOrigin-RevId: 522287217
2023-04-06 02:54:16 -07:00
MediaPipe TeamandCopybara-Service a151e6485a Internal change
PiperOrigin-RevId: 522287183
2023-04-06 02:50:05 -07:00
MediaPipe TeamandCopybara-Service 3c083f5d2b Internal change
PiperOrigin-RevId: 522282159
2023-04-06 02:18:58 -07:00
MediaPipe TeamandCopybara-Service 8d8ab9a972 Internal change
PiperOrigin-RevId: 522275233
2023-04-06 01:35:59 -07:00
MediaPipe TeamandCopybara-Service 22186299c4 Internal change
PiperOrigin-RevId: 522263621
2023-04-06 00:21:14 -07:00
MediaPipe TeamandCopybara-Service d05508cb7b Internal change
PiperOrigin-RevId: 522260226
2023-04-06 00:00:50 -07:00
MediaPipe TeamandCopybara-Service 56552dbfb5 Internal change
PiperOrigin-RevId: 522255364
2023-04-05 23:28:50 -07:00
MediaPipe TeamandCopybara-Service 12ecc8139f Internal change
PiperOrigin-RevId: 522255287
2023-04-05 23:24:55 -07:00
MediaPipe TeamandCopybara-Service 56b3cd4350 Internal change
PiperOrigin-RevId: 522253757
2023-04-05 23:14:45 -07:00
MediaPipe TeamandCopybara-Service 0067a1b5c2 Internal changes
PiperOrigin-RevId: 522248624
2023-04-05 22:37:11 -07:00
MediaPipe TeamandCopybara-Service 5a1a9269e6 Internal Changes
PiperOrigin-RevId: 522247775
2023-04-05 22:31:13 -07:00
Copybara-Service 7455022980 Merge pull request #4165 from kinaryml:audio-record-api-python
PiperOrigin-RevId: 522240683
2023-04-05 21:46:20 -07:00
MediaPipe TeamandCopybara-Service 7ae4d0175a CL will fix the typos in the tasks files
PiperOrigin-RevId: 522240681
2023-04-05 21:42:19 -07:00
MediaPipe TeamandCopybara-Service d5def9e24d Image segmenter output both confidence masks and category mask optionally.
PiperOrigin-RevId: 522227345
2023-04-05 20:34:18 -07:00
Hadon NashandCopybara-Service 7fe87936e5 Internal change
PiperOrigin-RevId: 522206591
2023-04-05 18:15:29 -07:00
neilblaze 07c0756b5d fixed errors in docs
Signed-off-by: Pratyay Banerjee <[email protected]>
2023-04-06 05:19:13 +05:30
Sebastian SchmidtandCopybara-Service f67007d077 Remove platform information for x86
PiperOrigin-RevId: 522182423
2023-04-05 16:21:24 -07:00
MediaPipe TeamandCopybara-Service 065d750781 Add VEC32F4 support to ImageFrame
PiperOrigin-RevId: 522153305
2023-04-05 14:25:17 -07:00
MediaPipe TeamandCopybara-Service 6605f551e7 Object Detector add batch_size and train_data to get_steps_per_epoch.
PiperOrigin-RevId: 522149938
2023-04-05 14:11:29 -07:00
MediaPipe TeamandCopybara-Service 5615c1e459 Delete duplicate public APIs in object detector
PiperOrigin-RevId: 522098326
2023-04-05 10:58:08 -07:00
MediaPipe TeamandCopybara-Service c5bd34ddb0 Internal change
PiperOrigin-RevId: 522014435
2023-04-05 04:25:25 -07:00
MediaPipe TeamandCopybara-Service 425a3ee3f6 Internal change
PiperOrigin-RevId: 521993439
2023-04-05 02:26:02 -07:00
MediaPipe TeamandCopybara-Service 91264eab1f Internal change
PiperOrigin-RevId: 521982139
2023-04-05 01:20:21 -07:00
MediaPipe TeamandCopybara-Service 05801b9945 Internal change
PiperOrigin-RevId: 521981387
2023-04-05 01:16:13 -07:00
MediaPipe TeamandCopybara-Service 46f9270788 Internal change
PiperOrigin-RevId: 521980958
2023-04-05 01:10:15 -07:00
MediaPipe TeamandCopybara-Service 7417e48da4 Internal change
PiperOrigin-RevId: 521970274
2023-04-05 00:05:39 -07:00
Kinar RandGitHub 1068755d2c Merge branch 'master' into interactive-segmenter-python 2023-04-05 10:51:45 +05:30
MediaPipe TeamandCopybara-Service 1990fe00d3 Internal change
PiperOrigin-RevId: 521949598
2023-04-04 21:35:10 -07:00
MediaPipe TeamandCopybara-Service 7cb8c647ca Internal change
PiperOrigin-RevId: 521948037
2023-04-04 21:24:55 -07:00
MediaPipe TeamandCopybara-Service 190be2e1bd Internal change
PiperOrigin-RevId: 521911790
2023-04-04 17:44:29 -07:00
MediaPipe TeamandCopybara-Service f8b2aa0633 Internal change
PiperOrigin-RevId: 521909998
2023-04-04 17:35:57 -07:00
MediaPipe TeamandCopybara-Service 9554836145 Update java image segmenter to always output confidence masks and optionally output category mask.
PiperOrigin-RevId: 521852718
2023-04-04 13:39:33 -07:00
MediaPipe TeamandCopybara-Service 55bcfcb4f5 Internal change
PiperOrigin-RevId: 521834742
2023-04-04 12:30:54 -07:00
Sebastian SchmidtandCopybara-Service a98f6bf231 FaceDetector Web API
PiperOrigin-RevId: 521816795
2023-04-04 11:23:00 -07:00
MediaPipe TeamandCopybara-Service 33cad24a5a Update java image segmenter to always output confidence masks and optionally output category mask.
PiperOrigin-RevId: 521804641
2023-04-04 10:41:59 -07:00
Copybara-Service 7c2930102d Merge pull request #4192 from kinaryml:face-stylizer-python
PiperOrigin-RevId: 521781683
2023-04-04 09:18:19 -07:00
MediaPipe TeamandCopybara-Service 65a98be809 Fixed comment and added note.
PiperOrigin-RevId: 521772542
2023-04-04 08:40:04 -07:00
MediaPipe TeamandCopybara-Service ec1d84aff7 Internal change
PiperOrigin-RevId: 521718577
2023-04-04 03:59:47 -07:00
MediaPipe TeamandCopybara-Service e95f465d58 Internal change
PiperOrigin-RevId: 521716263
2023-04-04 03:44:56 -07:00
MediaPipe TeamandCopybara-Service 53fa35e40c Add FrameBuffer view on ImageFrame.
PiperOrigin-RevId: 521689386
2023-04-04 01:19:07 -07:00
MediaPipe TeamandCopybara-Service 367ccbfdf3 update ImageSegmenterGraph to always output confidence mask and optionally output category mask
PiperOrigin-RevId: 521679910
2023-04-04 00:25:22 -07:00
Joe FernandezandCopybara-Service c31a4681e5 Fix left navigation mediapipe.dev docs legacy solutions web pages
PiperOrigin-RevId: 521609493
2023-04-03 17:44:07 -07:00
MediaPipe TeamandCopybara-Service 7c7eb74ef2 Internal change
PiperOrigin-RevId: 521586389
2023-04-03 16:00:24 -07:00
jqtang 3c05df9c46 Merge pull request #4235 from priankakariatyml:ios-object-detection-containers
PiperOrigin-RevId: 521553151
2023-04-03 15:12:06 -07:00
Sebastian SchmidtandCopybara-Service e84799ee37 Internal change
PiperOrigin-RevId: 521483663
2023-04-03 09:45:42 -07:00
Prianka Liz Kariat 048cc51e13 Added new line 2023-04-03 20:15:35 +05:30
Prianka Liz Kariat 1ab9b138ef Added MPPObjectDetectorOptions 2023-04-03 20:14:41 +05:30
Prianka Liz Kariat 67fcf9196e Added MPPObjectDetectionResult 2023-04-03 20:14:26 +05:30
Prianka Liz Kariat 4943029d62 Added MPPDetectionHelpers 2023-04-03 20:11:44 +05:30
Prianka Liz Kariat 9421249de1 Added MPPDetection 2023-04-03 20:07:45 +05:30
MediaPipe TeamandCopybara-Service cfe91f3c8c Internal change
PiperOrigin-RevId: 521424672
2023-04-03 04:46:59 -07:00
MediaPipe TeamandCopybara-Service 4a490cd27c This CL fixes the multiple typos in the new task api solution
PiperOrigin-RevId: 521407588
2023-04-03 03:05:29 -07:00
MediaPipe TeamandCopybara-Service b5bbed8ebb Internal change
PiperOrigin-RevId: 521406957
2023-04-03 03:01:23 -07:00
MediaPipe TeamandCopybara-Service 696bedcaa1 Internal change
PiperOrigin-RevId: 521327449
2023-04-02 17:43:47 -07:00
MediaPipe TeamandCopybara-Service 1fa9b2c985 Internal change
PiperOrigin-RevId: 521279971
2023-04-02 08:32:09 -07:00
MediaPipe TeamandCopybara-Service 50a49fd16c Internal change
PiperOrigin-RevId: 521226781
2023-04-01 22:29:07 -07:00
MediaPipe TeamandCopybara-Service d9f940f8b2 Model Maker object detector change learning_rate_boundaries to learning_rate_epoch_boundaries.
PiperOrigin-RevId: 521024056
2023-03-31 15:19:26 -07:00
Jiuqiang TangandCopybara-Service 7f9fd4f154 Add the minimum version number requirement to sounddevice in requirements.txt.
PiperOrigin-RevId: 520998845
2023-03-31 13:27:42 -07:00
Sebastian SchmidtandCopybara-Service c40e0fb6d5 Internal change
PiperOrigin-RevId: 520956127
2023-03-31 10:33:27 -07:00
MediaPipe TeamandCopybara-Service 1ff80f906c draw mouth to shoulder line after connection, to align with python viz code
PiperOrigin-RevId: 520935390
2023-03-31 09:05:10 -07:00
MediaPipe TeamandCopybara-Service 4dcb9a2201 Internal change
PiperOrigin-RevId: 520875109
2023-03-31 03:06:43 -07:00
MediaPipe TeamandCopybara-Service 733216da7e Use "x86_32" instead of "i386" for Bazel CPU ID -- they are currently synonyms, but "i386" is likely to be deprecated and removed in the future.
PiperOrigin-RevId: 520865783
2023-03-31 02:06:53 -07:00
MediaPipe TeamandCopybara-Service 5fe4e1ad0e Internal change
PiperOrigin-RevId: 520861522
2023-03-31 01:39:07 -07:00
Alan KellyandCopybara-Service 6a6786673e Do not explicitly set XNNPACK delegate flags.
XNNPACK is now enabled by default for all types so the behaviour remains identical.

PiperOrigin-RevId: 520855384
2023-03-31 00:59:29 -07:00
MediaPipe TeamandCopybara-Service 88d68341de Migrate face stylizer model files to GCS to use downloadable models in model maker.
PiperOrigin-RevId: 520848629
2023-03-31 00:11:51 -07:00
MediaPipe TeamandCopybara-Service 4ce87866da Add landmarks smoothing filter when requested face num is 1.
PiperOrigin-RevId: 520825046
2023-03-30 21:30:24 -07:00
MediaPipe TeamandCopybara-Service ea9083c89d visualize blaze pose for controlnet
PiperOrigin-RevId: 520823020
2023-03-30 21:20:06 -07:00
Sebastian SchmidtandCopybara-Service 145fc1ed38 Don't overwrite detection options if not specified in setOptions()
PiperOrigin-RevId: 520790479
2023-03-30 17:56:05 -07:00
Jiuqiang TangandCopybara-Service 508c72ddc9 Internal changes
PiperOrigin-RevId: 520775988
2023-03-30 16:45:34 -07:00
MediaPipe TeamandCopybara-Service 1eeb89e95f Add the model configuration and training hyperparameters for BlazeFaceStylizer.
PiperOrigin-RevId: 520767282
2023-03-30 16:07:11 -07:00
MediaPipe TeamandCopybara-Service a4923ca7aa Internal compatible_with change
PiperOrigin-RevId: 520751319
2023-03-30 15:01:09 -07:00
MediaPipe TeamandCopybara-Service f8f7126bdd Internal change
PiperOrigin-RevId: 520726897
2023-03-30 13:27:19 -07:00
MediaPipe TeamandCopybara-Service d43579fe3e Internal change
PiperOrigin-RevId: 520717805
2023-03-30 12:56:44 -07:00
MediaPipe TeamandCopybara-Service 99ba7dd787 Rewrite audio buffer conversion in Eigen primitives
PiperOrigin-RevId: 520717550
2023-03-30 12:52:22 -07:00
Jiuqiang TangandCopybara-Service 984073bf73 Fix the "'<>' with anonymous inner classes is not supported" error.
PiperOrigin-RevId: 520705926
2023-03-30 12:05:08 -07:00
MediaPipe TeamandCopybara-Service d7fd5b0cf5 Fix incorrect rotation handling in C++ vision tasks
PiperOrigin-RevId: 520670536
2023-03-30 10:05:25 -07:00
MediaPipe TeamandCopybara-Service 0e951b8add Internal change
PiperOrigin-RevId: 520636929
2023-03-30 07:48:07 -07:00
MediaPipe TeamandCopybara-Service 15d81576aa Change getLabels method to public
PiperOrigin-RevId: 520537239
2023-03-29 22:12:52 -07:00
Sebastian SchmidtandCopybara-Service f9eb3defa0 Internal change
PiperOrigin-RevId: 520523622
2023-03-29 21:04:07 -07:00
Sebastian SchmidtandCopybara-Service ac52859f1d Gracefully fail resource path lookup for Python on Windows
PiperOrigin-RevId: 520513921
2023-03-29 20:14:29 -07:00
Sebastian SchmidtandCopybara-Service ee1807d8e3 Don't use Bazel runfiles on for Python on Windows
PiperOrigin-RevId: 520504539
2023-03-29 19:10:07 -07:00
Sebastian SchmidtandCopybara-Service eaa708a18d Enable TextClassifier and TextEmbedder on Windows Python
PiperOrigin-RevId: 520498308
2023-03-29 18:26:53 -07:00
Sebastian SchmidtandCopybara-Service df67b35348 Internal change
PiperOrigin-RevId: 520486540
2023-03-29 17:27:16 -07:00
Sebastian SchmidtandCopybara-Service 6be4aedcf7 Use mediapipe_proto_library for all MediaPipe Protos
PiperOrigin-RevId: 520451054
2023-03-29 15:00:59 -07:00
MediaPipe TeamandCopybara-Service 0c8f691a36 Internal changes
PiperOrigin-RevId: 520411557
2023-03-29 12:35:20 -07:00
MediaPipe TeamandCopybara-Service 8e9207a7de Internal change
PiperOrigin-RevId: 520373147
2023-03-29 10:24:10 -07:00
Sebastian SchmidtandCopybara-Service b3d999704f FaceStylizer Java API
PiperOrigin-RevId: 520344417
2023-03-29 08:37:50 -07:00
MediaPipe TeamandCopybara-Service 316bd05e86 Internal change
PiperOrigin-RevId: 520263349
2023-03-29 01:35:04 -07:00
Sebastian SchmidtandCopybara-Service dc132a5629 Don't use Bazel runfiles on for Python on Windows
PiperOrigin-RevId: 520174325
2023-03-28 17:04:16 -07:00
MediaPipe TeamandCopybara-Service 9ee02481b1 Fix object detector pascal voc dataloader issue
PiperOrigin-RevId: 520160549
2023-03-28 16:06:44 -07:00
Jiuqiang TangandCopybara-Service 1e77725a15 Add necessary java lite proto source code to mediapipe tasks aar for the face landmarker task.
PiperOrigin-RevId: 520143851
2023-03-28 15:01:50 -07:00
Sebastian SchmidtandCopybara-Service 62d86494b0 Allow users to pass canvas: undefined
PiperOrigin-RevId: 520142520
2023-03-28 14:56:19 -07:00
Jiuqiang TangandCopybara-Service bda2639376 Switch to use the isPresent() API since the isEmpty() is only available since java 11: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Optional.html#isEmpty().
PiperOrigin-RevId: 520099308
2023-03-28 12:20:55 -07:00
Sebastian SchmidtandCopybara-Service 5c295da6ff Return custom error if model download fails
PiperOrigin-RevId: 520066065
2023-03-28 10:30:03 -07:00
Jiuqiang TangandCopybara-Service d4ec485971 Add face landmarker and face geometry java lite proto source code into mediapipe tasks AAR.
PiperOrigin-RevId: 520049667
2023-03-28 09:29:45 -07:00
MediaPipe TeamandCopybara-Service b4e1f0236a Add EndLoopImageCalculator
PiperOrigin-RevId: 520033132
2023-03-28 08:20:56 -07:00
MediaPipe TeamandCopybara-Service 0ea7b220f4 Add a function to convert CoreAudio buffers into a MediaPipe time series matrix
PiperOrigin-RevId: 519968274
2023-03-28 02:36:36 -07:00
MediaPipe TeamandCopybara-Service a18a62ef04 Make AnnotationOverlayCalculator compatible with GLES2/WebGL1 by using GL_RGB as internal format instead of GL_RGB8 for the texture that OpenCV renders into.
PiperOrigin-RevId: 519933934
2023-03-27 23:13:16 -07:00
Jiuqiang TangandCopybara-Service 94db96fa5e Add MatrixDataProto.java to mediapipe aar.
PiperOrigin-RevId: 519925872
2023-03-27 22:22:17 -07:00
Sebastian SchmidtandCopybara-Service 105e7b7467 Add no-copy Image getter for JNIS
PiperOrigin-RevId: 519909198
2023-03-27 20:59:23 -07:00
Copybara-Service 59b3150fff Merge pull request #4194 from priankakariatyml:ios-image-classifier-tests
PiperOrigin-RevId: 519907148
2023-03-27 20:46:18 -07:00
MediaPipe TeamandCopybara-Service 004265bbbd remove the check that data streams need to be > 0 since we have a use case of:
input_stream: "ALLOW:"
output_stream: "STATE_CHANGE"
PiperOrigin-RevId: 519891414
2023-03-27 19:14:24 -07:00
MediaPipe TeamandCopybara-Service 5f8831660f Internal MediaPipe Tasks change.
PiperOrigin-RevId: 519878741
2023-03-27 17:55:28 -07:00
Liam Miller-CushonandCopybara-Service 2d553a57e8 Update path to Doclava
PiperOrigin-RevId: 519850747
2023-03-27 15:51:26 -07:00
Jiuqiang TangandCopybara-Service c3b4fa5627 Expose face detector and face landmarker as public MediaPipe Tasks Python API.
PiperOrigin-RevId: 519822084
2023-03-27 14:03:21 -07:00
Sebastian SchmidtandCopybara-Service 7dd5f9b6c6 Add face_landmarker to vision types
PiperOrigin-RevId: 519815735
2023-03-27 13:39:51 -07:00
MediaPipe TeamandCopybara-Service 1fdc82d7ec Internal change
PiperOrigin-RevId: 519805707
2023-03-27 13:02:37 -07:00
Sebastian SchmidtandCopybara-Service e2f2acca5e Update WASM files for alpha-6 release
PiperOrigin-RevId: 519738578
2023-03-27 09:09:08 -07:00
MediaPipe TeamandCopybara-Service 7fdec2ecdf FaceLandmarker Java API
PiperOrigin-RevId: 519704560
2023-03-27 06:35:03 -07:00
MediaPipe TeamandCopybara-Service 9f888435b7 Internal change
PiperOrigin-RevId: 519674123
2023-03-27 03:40:24 -07:00
MediaPipe TeamandCopybara-Service 5ccf986513 This CL fixes the multiple typos in the new task api solution
PiperOrigin-RevId: 519634122
2023-03-26 23:57:38 -07:00
kinaryml e364aeb359 Revised Interactive Segmenter API and added more tests 2023-03-25 00:44:30 -07:00
Jiuqiang TangandCopybara-Service 55b9bd9d5e Internal change
PiperOrigin-RevId: 519224694
2023-03-24 13:33:47 -07:00
MediaPipe TeamandCopybara-Service 97c271644a Open Source Object Detector
PiperOrigin-RevId: 519201221
2023-03-24 11:53:18 -07:00
Sebastian SchmidtandCopybara-Service 58a98bc7da Add FaceLandmarker Web API
PiperOrigin-RevId: 519198210
2023-03-24 11:41:24 -07:00
Jiuqiang TangandCopybara-Service d6256362ec Internal change
PiperOrigin-RevId: 519186057
2023-03-24 10:56:50 -07:00
Sebastian SchmidtandCopybara-Service 9f6b2cd577 Add convertFromClassifications() helper
PiperOrigin-RevId: 519181016
2023-03-24 10:39:17 -07:00
Sebastian SchmidtandCopybara-Service cec878df2b Add Matrix output type
PiperOrigin-RevId: 519158476
2023-03-24 09:11:03 -07:00
MediaPipe TeamandCopybara-Service 53e4e92505 Support single channel golden images.
PiperOrigin-RevId: 519158051
2023-03-24 09:06:41 -07:00
MediaPipe TeamandCopybara-Service c26965c842 Internal change
PiperOrigin-RevId: 519150182
2023-03-24 08:34:11 -07:00
MediaPipe TeamandCopybara-Service 712ea6f15b Internal change
PiperOrigin-RevId: 519013105
2023-03-23 18:08:55 -07:00
Hadon NashandCopybara-Service 8a55f11952 Internal change
PiperOrigin-RevId: 519010016
2023-03-23 17:51:26 -07:00
Sebastian SchmidtandCopybara-Service 6aab4e013d Typo Fix
PiperOrigin-RevId: 518994598
2023-03-23 16:35:23 -07:00
MediaPipe TeamandCopybara-Service 27a5a6d433 Internal Changes
PiperOrigin-RevId: 518976200
2023-03-23 15:20:50 -07:00
MediaPipe TeamandCopybara-Service 948e17f404 Internal change
PiperOrigin-RevId: 518906661
2023-03-23 10:57:43 -07:00
Sebastian SchmidtandCopybara-Service 7b20a8e056 Update the WASM files for alpha5 Web release
PiperOrigin-RevId: 518905561
2023-03-23 10:53:38 -07:00
Sebastian SchmidtandCopybara-Service 8e5eadbd4e Add WebGLTexture output for ImageSegmenter
PiperOrigin-RevId: 518886135
2023-03-23 09:47:35 -07:00
Sebastian SchmidtandCopybara-Service 1c9e6894f3 Allow users to pass canvas element
PiperOrigin-RevId: 518870611
2023-03-23 08:46:21 -07:00
Prianka Liz Kariat 58adb69c44 Removed unwanted iOS tests 2023-03-23 20:16:30 +05:30
Prianka Liz Kariat 8682a3ffd9 Updated formatted 2023-03-23 19:58:03 +05:30
Prianka Liz Kariat da8b60700b Added flow limiter calculator to iOS vision tasks 2023-03-23 19:53:35 +05:30
Prianka Liz Kariat f51736e32d Added flow limiting for live stream mode in MPPImageClassifier 2023-03-23 19:52:14 +05:30
Prianka Liz Kariat 59e0b1ba74 Added stream info for some modes in MPPImageClassifier 2023-03-23 19:51:25 +05:30
Prianka Liz Kariat aa760855ee Updated formatting 2023-03-23 19:48:50 +05:30
Prianka Liz Kariat 1685664bdb Fixed formatting 2023-03-23 19:45:53 +05:30
Prianka Liz Kariat 55483776ab Fixed Issue with Flow Limiter in MPPTaskInfo 2023-03-23 19:44:56 +05:30
Prianka Liz Kariat d4b60a781e Added MPPImageClassifier Objective C Tests 2023-03-23 18:44:01 +05:30
Prianka Liz Kariat 48190e6600 Updated method signatures in MPPImage+TestUtils 2023-03-23 18:43:30 +05:30
Prianka Liz Kariat 8077743bfc Linked in Opencv ios framework with vision tasks. 2023-03-23 18:42:57 +05:30
Prianka Liz Kariat 1904632282 Fixed incorrect method call for image mode 2023-03-23 18:42:12 +05:30
Prianka Liz Kariat 960e7a6283 Fixed incorrect method call in MPPImageClassifier for Image Mode 2023-03-23 18:41:50 +05:30
Prianka Liz Kariat 1c4be91a3a Fixed stream names in MPPImageClassifier 2023-03-23 18:41:27 +05:30
Prianka Liz Kariat ddce041725 Fixed incorrect stride value in MPPImageUtils 2023-03-23 18:40:53 +05:30
Prianka Liz Kariat 3227635ea0 Fixed bug in roi assignment 2023-03-23 18:40:15 +05:30
Prianka Liz Kariat 6e62c113fb Fixed iOS running mode display strings 2023-03-23 18:39:54 +05:30
MediaPipe TeamandCopybara-Service 5998e96eed Internal change
PiperOrigin-RevId: 518814155
2023-03-23 03:35:43 -07:00
MediaPipe TeamandCopybara-Service 58fa1e2ec3 Internal change
PiperOrigin-RevId: 518813508
2023-03-23 03:31:46 -07:00
kinaryml 6304756c93 Added Interactive Segmenter Python API and some tests 2023-03-23 02:00:18 -07:00
Prianka Liz Kariat 2e4e17d837 Added MPPImage Utils for tests 2023-03-23 13:13:03 +05:30
MediaPipe TeamandCopybara-Service eac6348fd3 Open-sources a LanguageDetector C++ API.
PiperOrigin-RevId: 518758730
2023-03-22 21:54:35 -07:00
kinaryml 29a4041353 Fixed a typo in docstring 2023-03-22 21:51:45 -07:00
kinaryml ca18b95510 Updated BUILD 2023-03-22 21:50:40 -07:00
kinaryml 8cba65c229 Updated BUILD 2023-03-22 21:50:07 -07:00
kinaryml 613bcf99f4 Removed model 2023-03-22 21:49:11 -07:00
kinaryml c0320b556c Removed unit tests 2023-03-22 21:47:38 -07:00
kinaryml da70497f35 Updated Face Stylizer implementation and tests 2023-03-22 21:15:04 -07:00
Kinar RandGitHub 3afe4cafc4 Merge branch 'master' into face-stylizer-python 2023-03-23 09:27:52 +05:30
Sebastian SchmidtandCopybara-Service 1a7be8a4c1 Internal change
PiperOrigin-RevId: 518747623
2023-03-22 20:44:34 -07:00
MediaPipe TeamandCopybara-Service 37111e8fa5 Internal change
PiperOrigin-RevId: 518657355
2023-03-22 13:29:29 -07:00
Sebastian SchmidtandCopybara-Service eda8cb6b42 Typo fix
PiperOrigin-RevId: 518603982
2023-03-22 10:20:06 -07:00
MediaPipe TeamandCopybara-Service cb2ee87705 Internal change
PiperOrigin-RevId: 518591192
2023-03-22 09:34:23 -07:00
Jiuqiang TangandCopybara-Service 21e0ff3d4e Skip unnecessary cpu<->gpu conversion if the input and output are already on the same storage.
PiperOrigin-RevId: 518573284
2023-03-22 08:22:38 -07:00
MediaPipe TeamandCopybara-Service 18b4caa7f3 Internal change
PiperOrigin-RevId: 518559368
2023-03-22 07:14:18 -07:00
Kinar RandGitHub 3c39aca52b Merge branch 'google:master' into audio-record-api-python 2023-03-22 11:46:28 +05:30
kinaryml 444cd00ee6 Moved audio_record.py to tasks/python/audio/core 2023-03-21 23:15:55 -07:00
Copybara-Service 5d2a719b54 Merge pull request #4027 from kinaryml:cosine-sim-python
PiperOrigin-RevId: 518431807
2023-03-21 17:59:48 -07:00
Sebastian SchmidtandCopybara-Service 788e8d8777 Fix crash when FaceLandmarker does not return a result
PiperOrigin-RevId: 518410060
2023-03-21 16:21:58 -07:00
Sebastian SchmidtandCopybara-Service bbd21e9a6d Add the FaceStylizer Web API
PiperOrigin-RevId: 518409812
2023-03-21 16:17:28 -07:00
Jiuqiang TangandCopybara-Service b71e1d14d3 Add missing dependency library targets to mediapipe_task_aar.
PiperOrigin-RevId: 518384666
2023-03-21 14:38:11 -07:00
Sebastian SchmidtandCopybara-Service 8bbf2621a4 Typo fix
PiperOrigin-RevId: 518380030
2023-03-21 14:31:49 -07:00
Sebastian SchmidtandCopybara-Service a5fc1d4baf Use Uint8ClampedArray for pixel output
PiperOrigin-RevId: 518362677
2023-03-21 13:22:10 -07:00
MediaPipe TeamandCopybara-Service 7d26daf723 Make sure calling GraphTextureFrame::getTextureName()+::release() on a non-GL thread doesn't result in a crash.
PiperOrigin-RevId: 518359778
2023-03-21 13:12:03 -07:00
MediaPipe TeamandCopybara-Service 88effb19e5 Add build variants for _gms to some MediaPipe libraries that use TFLite
This change alters some cc_library to cc_library_with_tflite to add _gms variants to some select MediaPipe libraries. This CL also makes minimal changes in the code to make the _gms variants buildable.

PiperOrigin-RevId: 518336242
2023-03-21 11:48:09 -07:00
Jiuqiang TangandCopybara-Service 384f77b5c3 Fix the proto src file names.
PiperOrigin-RevId: 518332541
2023-03-21 11:34:39 -07:00
MediaPipe TeamandCopybara-Service c2a3e99545 Internal change
PiperOrigin-RevId: 518330697
2023-03-21 11:29:04 -07:00
MediaPipe TeamandCopybara-Service 2be66e8eb0 Add interactive segmenter java API
PiperOrigin-RevId: 518303391
2023-03-21 09:59:19 -07:00
Copybara-Service 6e0542c16a Merge pull request #4141 from priankakariatyml:ios-image-classifier-impl-files
PiperOrigin-RevId: 518138209
2023-03-20 19:05:37 -07:00
Jiuqiang TangandCopybara-Service 5a924669b7 Temporarily disabling checking whether the executor is not set in a subgraph node. This is a workaround to allow the MediaPipe InferenceCalculator to have its own executor.
PiperOrigin-RevId: 518117759
2023-03-20 17:10:55 -07:00
Sebastian SchmidtandCopybara-Service 2651d30ebf Add ImageData output to GraphRunner
PiperOrigin-RevId: 517994561
2023-03-20 09:46:27 -07:00
Copybara-Service 54e4dfc853 Merge pull request #4177 from kinaryml:face-landmarker-python
PiperOrigin-RevId: 517979421
2023-03-20 08:47:47 -07:00
MediaPipe TeamandCopybara-Service 47fa1a9578 Internal change
PiperOrigin-RevId: 517886450
2023-03-20 00:25:47 -07:00
MediaPipe TeamandCopybara-Service 524acaaaa7 internal change
PiperOrigin-RevId: 517823184
2023-03-19 16:07:32 -07:00
Jiuqiang TangandCopybara-Service f06e4224b8 Allow ModelResourcesCalculator to use file_pointer_meta.
PiperOrigin-RevId: 517742722
2023-03-19 01:33:52 -07:00
MediaPipe TeamandCopybara-Service 2e5ea174fe Internal change
PiperOrigin-RevId: 517597924
2023-03-18 00:29:59 -07:00
Jiuqiang TangandCopybara-Service 6634d22161 Add LabelMapProto.java source code to MediaPipe AAR.
PiperOrigin-RevId: 517563190
2023-03-17 19:49:53 -07:00
Jiuqiang TangandCopybara-Service 6785bcc47d Add java_package and java_outer_classname to label_map.proto.
PiperOrigin-RevId: 517563000
2023-03-17 19:45:45 -07:00
Jiuqiang TangandCopybara-Service 065f1f38aa Fix the vision tasks aar build rule to solve the "cannot find symbol" error:
```
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java:28: error: cannot find symbol
import com.google.mediapipe.tasks.TensorsToSegmentationCalculatorOptionsProto;
```

PiperOrigin-RevId: 517542284
2023-03-17 17:02:22 -07:00
Jiuqiang TangandCopybara-Service 0805d61bfe Add the source code TensorsToSegmentationCalculatorOptionsProto.java into tasks core's maven package.
PiperOrigin-RevId: 517516701
2023-03-17 15:01:33 -07:00
MediaPipe TeamandCopybara-Service 3dede1a9a5 Add label_map filtering into filter_detection drishti calculator.
PiperOrigin-RevId: 517515046
2023-03-17 14:53:48 -07:00
MediaPipe TeamandCopybara-Service 1a456dcbf9 Internal change
PiperOrigin-RevId: 517466009
2023-03-17 11:40:26 -07:00
MediaPipe TeamandCopybara-Service 24217e2ead Internal change
PiperOrigin-RevId: 517322105
2023-03-16 22:50:22 -07:00
kinaryml 1ba285d916 Updated a method name in face_landmarker_test.py 2023-03-16 15:10:39 -07:00
Sebastian SchmidtandCopybara-Service 75bc44d42a Add a comment to macos_opencv so it can be easily replaced with sed
PiperOrigin-RevId: 517225588
2023-03-16 14:19:14 -07:00
MediaPipe TeamandCopybara-Service 560945ad39 Internal Changes
PiperOrigin-RevId: 517219631
2023-03-16 13:58:02 -07:00
kinaryml 94dba82284 Renamed a test method to use the plural form 2023-03-16 12:54:38 -07:00
kinaryml b36b0bb3e8 Updated API and tests 2023-03-16 12:51:19 -07:00
kinaryml 2753c79fde Removed MatrixData dataclass and used NumPy to represent Matrix 2023-03-16 11:50:07 -07:00
MediaPipe TeamandCopybara-Service a9e956baa1 Add more details to the invoke call trace.
It is always useful information to know if the TPU invoke is Async or not, and if the GPU invoke on the old path or new path.
This can make it obvious in the perfetto trace.

PiperOrigin-RevId: 517162515
2023-03-16 10:37:32 -07:00
MediaPipe TeamandCopybara-Service 3b66ac0623 Import the saved Keras models of BlazeFaceStylizer components into MediaPipe model maker.
PiperOrigin-RevId: 517044265
2023-03-16 00:44:06 -07:00
kinaryml 9aea1be6f9 Removed geometry pipeline calculator 2023-03-15 23:51:12 -07:00
Kinar RandGitHub 15d90bd325 Merge branch 'master' into face-landmarker-python 2023-03-16 12:17:59 +05:30
Copybara-Service d84ccbadad Merge pull request #4158 from kinaryml:face-detector-python
PiperOrigin-RevId: 516970627
2023-03-15 17:13:37 -07:00
MediaPipe TeamandCopybara-Service 8f1ce5fef6 Add quality test for InteractiveSegmenter
PiperOrigin-RevId: 516968294
2023-03-15 17:02:34 -07:00
MediaPipe TeamandCopybara-Service 61bcddc671 Add Interactive Segmenter MediaPipe Task
PiperOrigin-RevId: 516954589
2023-03-15 16:05:18 -07:00
Sebastian SchmidtandCopybara-Service 43082482f8 Remove framework:Cocoa again
PiperOrigin-RevId: 516928735
2023-03-15 14:24:15 -07:00
MediaPipe TeamandCopybara-Service 59962bed27 ImageSegmenterGraph set activation type from metadata, and remove the activation config in C++ ImageSegmenterOptions.
PiperOrigin-RevId: 516893115
2023-03-15 12:13:00 -07:00
MediaPipe TeamandCopybara-Service a323825134 Internal change
PiperOrigin-RevId: 516882513
2023-03-15 11:35:49 -07:00
MediaPipe TeamandCopybara-Service 18d88c531a Internal MediaPipe Tasks change.
PiperOrigin-RevId: 516881879
2023-03-15 11:30:58 -07:00
MediaPipe TeamandCopybara-Service ce3cd94f45 Internal change
PiperOrigin-RevId: 516871638
2023-03-15 10:56:32 -07:00
kinaryml 80dd764605 Removed dummy packet creation and preserved face_geometry protobuf import 2023-03-15 10:52:16 -07:00
kinaryml 4a6015e65c Fixed some issues in the MatrixData container, revised the implementation and added more tests 2023-03-15 10:41:36 -07:00
kinaryml 06c37c6442 Updated mediapipe/python/BUILD and tests 2023-03-15 09:11:06 -07:00
Kinar RandGitHub 2af660321d Merge branch 'google:master' into face-landmarker-python 2023-03-15 21:22:35 +05:30
Fergus HendersonandCopybara-Service 04ffb8432e Fix typo.
PiperOrigin-RevId: 516834369
2023-03-15 08:41:40 -07:00
Sebastian SchmidtandCopybara-Service c8b56439af Fix typo
PiperOrigin-RevId: 516758040
2023-03-15 01:38:55 -07:00
Kinar RandGitHub 647db21fc3 Merge branch 'google:master' into face-landmarker-python 2023-03-15 11:03:15 +05:30
kinaryml d83f400b08 Updated API and tests 2023-03-14 22:32:39 -07:00
MediaPipe TeamandCopybara-Service f517eddce1 API for c++ ImageSegmenter to get labels
PiperOrigin-RevId: 516714139
2023-03-14 21:13:08 -07:00
MediaPipe TeamandCopybara-Service cafff14135 GeometryPipelineCalculator support single face landmarks input.
PiperOrigin-RevId: 516701488
2023-03-14 20:03:07 -07:00
MediaPipe TeamandCopybara-Service 141cf843ae Add getLabels to ImageSegmeter Java API
PiperOrigin-RevId: 516683339
2023-03-14 18:04:33 -07:00
Yuqi LiandCopybara-Service 51d9640d88 Add metadata writer for image segmentation.
PiperOrigin-RevId: 516671364
2023-03-14 17:00:30 -07:00
MediaPipe TeamandCopybara-Service 9a89b47572 Rename *ModelFile to *File for methods of ModelAssetBundleResources.
PiperOrigin-RevId: 516667461
2023-03-14 16:42:32 -07:00
Jiuqiang TangandCopybara-Service cd2cc971bb Registering FaceGeometry proto.
PiperOrigin-RevId: 516663848
2023-03-14 16:28:48 -07:00
Sebastian SchmidtandCopybara-Service f60fe739d3 Internal change
PiperOrigin-RevId: 516656029
2023-03-14 15:57:01 -07:00
Sebastian SchmidtandCopybara-Service 3e952a5f1f Update Node version to 16.19.0
PiperOrigin-RevId: 516654979
2023-03-14 15:52:32 -07:00
Sebastian SchmidtandCopybara-Service ec3cd45d61 Add InteractiveSegmenter Web API
PiperOrigin-RevId: 516654090
2023-03-14 15:48:38 -07:00
MediaPipe TeamandCopybara-Service 6774794d02 Add the dataset module for face stylizer in model maker.
PiperOrigin-RevId: 516628350
2023-03-14 14:11:20 -07:00
Hadon NashandCopybara-Service ade31b567b Internal change
PiperOrigin-RevId: 516626371
2023-03-14 14:04:57 -07:00
MediaPipe TeamandCopybara-Service ed3e728bb8 Internal change
PiperOrigin-RevId: 516607678
2023-03-14 12:57:24 -07:00
Jiuqiang TangandCopybara-Service fef8b9cb58 Registering FaceGeometry proto.
PiperOrigin-RevId: 516597971
2023-03-14 12:21:02 -07:00
MediaPipe TeamandCopybara-Service 854ab25ee9 Internal change.
PiperOrigin-RevId: 516594221
2023-03-14 12:11:18 -07:00
MediaPipe TeamandCopybara-Service 8a41a5e44d Update models.
PiperOrigin-RevId: 516575530
2023-03-14 11:06:20 -07:00
Sebastian SchmidtandCopybara-Service 5bba1f245f Internal change
PiperOrigin-RevId: 516550977
2023-03-14 09:46:41 -07:00
MediaPipe TeamandCopybara-Service c895867427 Expose FrameBuffer view on GpuBufferStorageYuvImage.
PiperOrigin-RevId: 516546716
2023-03-14 09:30:25 -07:00
MediaPipe TeamandCopybara-Service 2659ea0392 Internal change
PiperOrigin-RevId: 516535124
2023-03-14 08:44:40 -07:00
Alan KellyandCopybara-Service bc641a22a8 Internal change
PiperOrigin-RevId: 516520860
2023-03-14 07:41:20 -07:00
kinaryml 23681cde0d Revised face landmarker implementation and tests 2023-03-14 00:37:32 -07:00
Kinar RandGitHub 4a7489cd3a Merge branch 'google:master' into face-landmarker-python 2023-03-14 11:27:52 +05:30
MediaPipe TeamandCopybara-Service 46ba1d8051 Use ExternalFile to set metadata of GeometryPipelineCalculator.
PiperOrigin-RevId: 516384491
2023-03-13 18:50:33 -07:00
Jiuqiang TangandCopybara-Service 57f106e0a7 Wait until the metal backend finishes its work in the TensorsToImageCalculator.
PiperOrigin-RevId: 516360846
2023-03-13 16:56:46 -07:00
MediaPipe TeamandCopybara-Service 89857f33a2 Make ImageToTensorCalculator use kGpuService optionally
PiperOrigin-RevId: 516358053
2023-03-13 16:48:49 -07:00
Chris McClanahanandCopybara-Service 1b4a835be0 Internal change
PiperOrigin-RevId: 516349788
2023-03-13 16:09:56 -07:00
Sebastian SchmidtandCopybara-Service cb4b0ea93d Disable OpenCL dependency for OpenCV
PiperOrigin-RevId: 516341303
2023-03-13 15:37:53 -07:00
Esha UbowejaandCopybara-Service 0f58d89992 Preserves all elements of BASE_HAND_RECTS input streams in HandAssociationCalculator.
PiperOrigin-RevId: 516339343
2023-03-13 15:32:35 -07:00
Sebastian SchmidtandCopybara-Service c32ddcb04c Add alwayslink to face_stylizer_graph
PiperOrigin-RevId: 516330940
2023-03-13 14:58:39 -07:00
Sebastian SchmidtandCopybara-Service eac2e337f6 Sort vision tasks in README.md
PiperOrigin-RevId: 516312229
2023-03-13 13:53:20 -07:00
Sebastian SchmidtandCopybara-Service 85600ca326 Add Keypoint and Region-of-interest
PiperOrigin-RevId: 516299794
2023-03-13 13:10:40 -07:00
Sebastian SchmidtandCopybara-Service 490d1a7516 Refactor Web code for InteractiveSegmenter
PiperOrigin-RevId: 516254891
2023-03-13 10:43:24 -07:00
Copybara-Service d6fb7c365e Merge pull request #4145 from lucifertrj:poseDocs
PiperOrigin-RevId: 516231033
2023-03-13 09:22:34 -07:00
kinaryml efae2830f1 Updated face landmarker implementation and tests 2023-03-13 08:46:41 -07:00
Jiuqiang TangandCopybara-Service 1d2041b992 Internal change
PiperOrigin-RevId: 516220827
2023-03-13 08:43:21 -07:00
MediaPipe TeamandCopybara-Service ffea85e470 Internal change
PiperOrigin-RevId: 516179167
2023-03-13 05:13:01 -07:00
kinaryml 89be4c7b64 Added some files for the face landmarker implementation 2023-03-12 16:09:04 -07:00
MediaPipe TeamandCopybara-Service 131be2169a Add FaceDetector Java API
PiperOrigin-RevId: 515913662
2023-03-11 13:14:47 -08:00
MediaPipe TeamandCopybara-Service 296ee33be5 Add FaceLandmarker C++ API
PiperOrigin-RevId: 515912777
2023-03-11 13:05:21 -08:00
kinaryml 78e48825ae Make create_audio_record not a static method 2023-03-11 07:39:34 -08:00
MediaPipe TeamandCopybara-Service c94de4032d Fix preprocess Callable typing
PiperOrigin-RevId: 515818356
2023-03-10 21:50:46 -08:00
Jiuqiang TangandCopybara-Service 3e8fd58400 Make createAudioRecord a class method not a static method.
PiperOrigin-RevId: 515740313
2023-03-10 14:34:32 -08:00
Sebastian SchmidtandCopybara-Service db779ba78f Use drishti_proto_library for libraries in mediapipe/gpu
PiperOrigin-RevId: 515708036
2023-03-10 12:27:06 -08:00
MediaPipe TeamandCopybara-Service c9bd4f5957 Internal change
PiperOrigin-RevId: 515706419
2023-03-10 12:22:27 -08:00
MediaPipe TeamandCopybara-Service c3a32d76be Update face geometry proto java package name.
PiperOrigin-RevId: 515696170
2023-03-10 11:45:35 -08:00
MediaPipe TeamandCopybara-Service 296c343332 Revise the Halide Bazel build rules to be cleaner and more correct. The initial version we landed happened to work (at least in most cases) but had known workarounds in it. AFAICT, the issue seemed to be that the MediaPipe config_setting() values we were using didn't always resolve to the right thing when detecting the host cpu (vs the compile-target cpu), which is critical for Halide. I fixed this by avoiding the use of the MediaPipe config_settings entirely, using instead the public Bazel settings hosted at https://github.com/bazelbuild/platforms.
This revision builds and runs //mediapipe/util/frame_buffer:all correctly on my mac x86-64 laptop; I haven't yet attempted to build or test on any other platform, so there may well still be glitches, but I think this is more fundamentally sound than what we had before.

PiperOrigin-RevId: 515682507
2023-03-10 10:59:17 -08:00
Kinar RandGitHub 9d82000148 Update BUILD 2023-03-11 00:04:07 +05:30
Kinar RandGitHub f56a3088e3 Update audio_record_test.py 2023-03-11 00:02:41 +05:30
kinaryml 9787056508 Added the AudioRecord API 2023-03-10 10:17:03 -08:00
MediaPipe TeamandCopybara-Service 05b505c8e2 Introduce api to disable service default initialization.
PiperOrigin-RevId: 515501608
2023-03-09 18:57:39 -08:00
Sebastian SchmidtandCopybara-Service ef4a8cde42 Solve iOS build error for gpu_buffer.cc
PiperOrigin-RevId: 515473643
2023-03-09 16:28:58 -08:00
MediaPipe TeamandCopybara-Service 2d8f937913 Improve docstring of image classifier model spec.
PiperOrigin-RevId: 515466722
2023-03-09 16:06:40 -08:00
kinaryml 7463e48fd4 Added some files necessary for the Face Stylizer implementation 2023-03-09 02:39:21 -08:00
kinaryml f48909cab6 Fixed score's data type 2023-03-09 02:13:34 -08:00
kinaryml 24114ec2fe Updated comment in test 2023-03-09 01:41:42 -08:00
kinaryml 022838a7f3 Added Face Detector implementation and tests 2023-03-09 01:36:39 -08:00
Tarun JainandGitHub 54e597216b np used in the program but not imported 2023-03-04 22:58:12 +05:30
Prianka Liz Kariat 87df23f0fa Updated formatting of MPPRunningMode.h 2023-03-03 14:40:00 +05:30
Prianka Liz Kariat 7b8d92ba47 Updated formatting in MPPVisionTaskRunner 2023-03-03 14:39:22 +05:30
Prianka Liz Kariat 412476eba1 Added inline function to return display name of MPPRunningMode 2023-03-03 14:39:03 +05:30
Prianka Liz Kariat f2dfa7f474 Updated documentation of MPPVisionTaskRunner 2023-03-03 12:33:31 +05:30
Prianka Liz Kariat 160fa424b5 Updated documentation of MPPTaskRunner 2023-03-03 12:33:17 +05:30
Prianka Liz Kariat fee66069ac Updated error with info about unsupported mirrored orientations in MPPVisionTaskRunner 2023-03-03 12:19:14 +05:30
Prianka Liz Kariat 6253966901 Updated comments in MPPTaskRunner to include note about mirrored orientations 2023-03-03 12:15:25 +05:30
Prianka Liz Kariat 289b3b20de Added methods for common functionality in MPPImageClassifier 2023-03-03 12:01:45 +05:30
Prianka Liz Kariat 8aaabe4a02 Updated comments in MPPVisionTaskRunner 2023-03-03 11:18:36 +05:30
Prianka Liz Kariat d577727698 Updated MPPImageClassifier 2023-03-03 11:11:07 +05:30
Prianka Liz Kariat af82dc5e17 Updated comments in MPPImageClassifier 2023-03-03 11:10:20 +05:30
Prianka Liz Kariat 4b54f7f45f Fixed comments in MPPVisionTaskRunner 2023-03-03 10:57:39 +05:30
Tarun JainandGitHub 38ec8ae842 link updated 2023-03-02 21:04:53 +05:30
Tarun JainandGitHub 6e7018b826 Merge branch 'google:master' into master 2023-03-02 21:01:21 +05:30
Prianka Liz Kariat 09fa23088a Added TODO 2023-03-02 20:01:48 +05:30
Prianka Liz Kariat 33a34de03b Updated method signature in MPPTaskRunner 2023-03-02 19:50:00 +05:30
Prianka Liz Kariat fe7bb92859 Added dependency for image format 2023-03-02 19:49:42 +05:30
Prianka Liz Kariat 6eafddb8e2 Added MPPImageClassifier 2023-03-02 19:46:20 +05:30
Prianka Liz Kariat ee6171c833 Added MPPImageClassifierOptionsHelpers 2023-03-02 19:46:11 +05:30
Prianka Liz Kariat abb140ed2e Added MPPImageClassifierResultHelpers 2023-03-02 19:45:58 +05:30
Prianka Liz Kariat c871fc58ec Updated build targets of vision packet creator and task runner 2023-03-02 19:43:46 +05:30
Prianka Liz Kariat c625d6afdc Added methods to MPPVisionPacketCreator 2023-03-02 19:43:22 +05:30
Prianka Liz Kariat dc393b0bd4 Added methods to MPPVisionTaskRunner 2023-03-02 19:43:01 +05:30
Prianka Liz Kariat b76ab37394 Removed unwanted imports 2023-03-02 19:37:24 +05:30
Prianka Liz Kariat 045050fc85 Changed method Updated method calls to process packet map in iOS text tasks 2023-03-02 19:36:34 +05:30
Prianka Liz Kariat 6d7f172e9f Changed return type of process method in MPPTaskRunner 2023-03-02 19:35:05 +05:30
Prianka Liz Kariat 87ba86ace2 Added method to send packet map to C++ task runner 2023-03-02 19:34:44 +05:30
Tarun JainandGitHub c73ff261ad Fix Broken Link in Object Detection solutions 2023-02-24 19:37:30 +05:30
Kinar RandGitHub 2dc790dbd9 Merge branch 'google:master' into cosine-sim-python 2023-01-25 00:25:23 +05:30
kinaryml ea77a7c25d Undo commenting out remaining tests 2022-11-17 14:06:30 -08:00
kinaryml 87238705dd Updated cosine similarity utility 2022-11-17 14:03:07 -08:00
823 changed files with 47612 additions and 9472 deletions
+1 -1
View File
@@ -1 +1 @@
5.2.0
6.1.1
+1 -1
View File
@@ -61,7 +61,7 @@ RUN pip3 install tf_slim
RUN ln -s /usr/bin/python3 /usr/bin/python
# Install bazel
ARG BAZEL_VERSION=5.2.0
ARG BAZEL_VERSION=6.1.1
RUN mkdir /bazel && \
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
+17
View File
@@ -199,3 +199,20 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
===========================================================================
For files under tasks/cc/text/language_detector/custom_ops/utils/utf/
===========================================================================
/*
* The authors of this software are Rob Pike and Ken Thompson.
* Copyright (c) 2002 by Lucent Technologies.
* Permission to use, copy, modify, and distribute this software for any
* purpose without fee is hereby granted, provided that this entire notice
* is included in all copies of any software which is or includes a copy
* or modification of this software and in all copies of the supporting
* documentation for such software.
* THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY
* REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY
* OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE.
*/
+14 -9
View File
@@ -6,6 +6,20 @@ nav_order: 1
![MediaPipe](https://mediapipe.dev/images/mediapipe_small.png)
----
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
*This notice and web page will be removed on June 1, 2023.*
----
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
--------------------------------------------------------------------------------
## Live ML anywhere
@@ -21,15 +35,6 @@ ML solutions for live and streaming media.
----
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023.*
*This notice and web page will be removed on April 3, 2023.*
----
## ML solutions in MediaPipe
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
+88 -85
View File
@@ -54,6 +54,76 @@ load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependen
rules_foreign_cc_dependencies()
http_archive(
name = "com_google_protobuf",
sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422",
strip_prefix = "protobuf-3.19.1",
urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"],
patches = [
"@//third_party:com_google_protobuf_fixes.diff"
],
patch_args = [
"-p1",
],
)
# Load Zlib before initializing TensorFlow and the iOS build rules to guarantee
# that the target @zlib//:mini_zlib is available
http_archive(
name = "zlib",
build_file = "@//third_party:zlib.BUILD",
sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
strip_prefix = "zlib-1.2.11",
urls = [
"http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
"http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
],
patches = [
"@//third_party:zlib.diff",
],
patch_args = [
"-p1",
],
)
# iOS basic build deps.
http_archive(
name = "build_bazel_rules_apple",
sha256 = "3e2c7ae0ddd181c4053b6491dad1d01ae29011bc322ca87eea45957c76d3a0c3",
url = "https://github.com/bazelbuild/rules_apple/releases/download/2.1.0/rules_apple.2.1.0.tar.gz",
patches = [
# Bypass checking ios unit test runner when building MP ios applications.
"@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
],
patch_args = [
"-p1",
],
)
load(
"@build_bazel_rules_apple//apple:repositories.bzl",
"apple_rules_dependencies",
)
apple_rules_dependencies()
load(
"@build_bazel_rules_swift//swift:repositories.bzl",
"swift_rules_dependencies",
)
swift_rules_dependencies()
load(
"@build_bazel_rules_swift//swift:extras.bzl",
"swift_rules_extra_dependencies",
)
swift_rules_extra_dependencies()
load(
"@build_bazel_apple_support//lib:repositories.bzl",
"apple_support_dependencies",
)
apple_support_dependencies()
# This is used to select all contents of the archives for CMake-based packages to give CMake access to them.
all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])"""
@@ -133,19 +203,6 @@ http_archive(
urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"],
)
http_archive(
name = "com_google_protobuf",
sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422",
strip_prefix = "protobuf-3.19.1",
urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"],
patches = [
"@//third_party:com_google_protobuf_fixes.diff"
],
patch_args = [
"-p1",
],
)
load("@//third_party/flatbuffers:workspace.bzl", flatbuffers = "repo")
flatbuffers()
@@ -155,6 +212,9 @@ http_archive(
urls = ["https://github.com/google/multichannel-audio-tools/archive/1f6b1319f13282eda6ff1317be13de67f4723860.zip"],
sha256 = "fe346e1aee4f5069c4cbccb88706a9a2b2b4cf98aeb91ec1319be77e07dd7435",
repo_mapping = {"@com_github_glog_glog" : "@com_github_glog_glog_no_gflags"},
# TODO: Fix this in AudioTools directly
patches = ["@//third_party:com_google_audio_tools_fixes.diff"],
patch_args = ["-p1"]
)
http_archive(
@@ -270,7 +330,7 @@ new_local_repository(
# For local MacOS builds, the path should point to an opencv@3 installation.
# If you edit the path here, you will also need to update the corresponding
# prefix in "opencv_macos.BUILD".
path = "/usr/local",
path = "/usr/local", # e.g. /usr/local/Cellar for HomeBrew
)
new_local_repository(
@@ -319,63 +379,6 @@ http_archive(
],
)
# Load Zlib before initializing TensorFlow and the iOS build rules to guarantee
# that the target @zlib//:mini_zlib is available
http_archive(
name = "zlib",
build_file = "@//third_party:zlib.BUILD",
sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
strip_prefix = "zlib-1.2.11",
urls = [
"http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
"http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
],
patches = [
"@//third_party:zlib.diff",
],
patch_args = [
"-p1",
],
)
# iOS basic build deps.
http_archive(
name = "build_bazel_rules_apple",
sha256 = "f94e6dddf74739ef5cb30f000e13a2a613f6ebfa5e63588305a71fce8a8a9911",
url = "https://github.com/bazelbuild/rules_apple/releases/download/1.1.3/rules_apple.1.1.3.tar.gz",
patches = [
# Bypass checking ios unit test runner when building MP ios applications.
"@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
],
patch_args = [
"-p1",
],
)
load(
"@build_bazel_rules_apple//apple:repositories.bzl",
"apple_rules_dependencies",
)
apple_rules_dependencies()
load(
"@build_bazel_rules_swift//swift:repositories.bzl",
"swift_rules_dependencies",
)
swift_rules_dependencies()
load(
"@build_bazel_rules_swift//swift:extras.bzl",
"swift_rules_extra_dependencies",
)
swift_rules_extra_dependencies()
load(
"@build_bazel_apple_support//lib:repositories.bzl",
"apple_support_dependencies",
)
apple_support_dependencies()
# More iOS deps.
http_archive(
@@ -499,8 +502,8 @@ cc_crosstool(name = "crosstool")
# Node dependencies
http_archive(
name = "build_bazel_rules_nodejs",
sha256 = "5aae76dced38f784b58d9776e4ab12278bc156a9ed2b1d9fcd3e39921dc88fda",
urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/5.7.1/rules_nodejs-5.7.1.tar.gz"],
sha256 = "94070eff79305be05b7699207fbac5d2608054dd53e6109f7d00d923919ff45a",
urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/5.8.2/rules_nodejs-5.8.2.tar.gz"],
)
load("@build_bazel_rules_nodejs//:repositories.bzl", "build_bazel_rules_nodejs_dependencies")
@@ -554,32 +557,32 @@ new_local_repository(
http_archive(
name = "linux_halide",
sha256 = "f62b2914823d6e33d18693f5b74484f274523bf5402ce51988e24393d123b375",
strip_prefix = "Halide-15.0.0-x86-64-linux",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.0/Halide-15.0.0-x86-64-linux-d7651f4b32f9dbd764f243134001f7554378d62d.tar.gz"],
sha256 = "d290fadf3f358c94aacf43c883de6468bb98883e26116920afd491ec0e440cd2",
strip_prefix = "Halide-15.0.1-x86-64-linux",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.1/Halide-15.0.1-x86-64-linux-4c63f1befa1063184c5982b11b6a2cc17d4e5815.tar.gz"],
build_file = "@//third_party:halide.BUILD",
)
http_archive(
name = "macos_x86_64_halide",
sha256 = "3d832aed942080ea89aa832462c68fbb906f3055c440b7b6d35093d7c52f6aab",
strip_prefix = "Halide-15.0.0-x86-64-osx",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.0/Halide-15.0.0-x86-64-osx-d7651f4b32f9dbd764f243134001f7554378d62d.tar.gz"],
sha256 = "48ff073ac1aee5c4aca941a4f043cac64b38ba236cdca12567e09d803594a61c",
strip_prefix = "Halide-15.0.1-x86-64-osx",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.1/Halide-15.0.1-x86-64-osx-4c63f1befa1063184c5982b11b6a2cc17d4e5815.tar.gz"],
build_file = "@//third_party:halide.BUILD",
)
http_archive(
name = "macos_arm_64_halide",
sha256 = "b1fad3c9810122b187303d7031d9e35fb43761f345d18cc4492c00ed5877f641",
strip_prefix = "Halide-15.0.0-arm-64-osx",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.0/Halide-15.0.0-arm-64-osx-d7651f4b32f9dbd764f243134001f7554378d62d.tar.gz"],
sha256 = "db5d20d75fa7463490fcbc79c89f0abec9c23991f787c8e3e831fff411d5395c",
strip_prefix = "Halide-15.0.1-arm-64-osx",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.1/Halide-15.0.1-arm-64-osx-4c63f1befa1063184c5982b11b6a2cc17d4e5815.tar.gz"],
build_file = "@//third_party:halide.BUILD",
)
http_archive(
name = "windows_halide",
sha256 = "5acf6fe161dd375856a2b43f4bb0a32815ba958b0585ee312c44e008aa7b0b64",
strip_prefix = "Halide-15.0.0-x86-64-windows",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.0/Halide-15.0.0-x86-64-windows-d7651f4b32f9dbd764f243134001f7554378d62d.zip"],
sha256 = "61fd049bd75ee918ac6c30d0693aac6048f63f8d1fc4db31001573e58eae8dae",
strip_prefix = "Halide-15.0.1-x86-64-windows",
urls = ["https://github.com/halide/Halide/releases/download/v15.0.1/Halide-15.0.1-x86-64-windows-4c63f1befa1063184c5982b11b6a2cc17d4e5815.zip"],
build_file = "@//third_party:halide.BUILD",
)
+2 -2
View File
@@ -16,11 +16,11 @@ py_binary(
srcs = ["build_java_api_docs.py"],
data = [
"//third_party/android/sdk:api/26.txt",
"//third_party/java/doclava/current:doclava.jar",
"//third_party/java/doclava:doclet.jar",
"//third_party/java/jsilver:jsilver_jar",
],
env = {
"DOCLAVA_JAR": "$(location //third_party/java/doclava/current:doclava.jar)",
"DOCLAVA_JAR": "$(location //third_party/java/doclava:doclet.jar)",
"JSILVER_JAR": "$(location //third_party/java/jsilver:jsilver_jar)",
},
deps = [
@@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/framework/framework_concepts/graphs_cpp
title: Building Graphs in C++
parent: Graphs
nav_order: 1
@@ -12,6 +13,12 @@ nav_order: 1
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
C++ graph builder is a powerful tool for:
* Building complex graphs
+6
View File
@@ -13,6 +13,12 @@ nav_order: 1
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
Each calculator is a node of a graph. We describe how to create a new
calculator, how to initialize a calculator, how to perform its calculations,
input and output streams, timestamps, and options. Each node in the graph is
@@ -14,6 +14,12 @@ has_toc: false
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## The basics
### Packet
+6
View File
@@ -13,6 +13,12 @@ nav_order: 5
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## Overview
MediaPipe supports calculator nodes for GPU compute and rendering, and allows combining multiple GPU nodes, as well as mixing them with CPU based calculator nodes. There exist several GPU APIs on mobile platforms (eg, OpenGL ES, Metal and Vulkan). MediaPipe does not attempt to offer a single cross-API GPU abstraction. Individual nodes can be written using different APIs, allowing them to take advantage of platform specific features when needed.
+6
View File
@@ -13,6 +13,12 @@ nav_order: 2
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## Graph
A `CalculatorGraphConfig` proto specifies the topology and functionality of a
+6
View File
@@ -13,6 +13,12 @@ nav_order: 3
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
Calculators communicate by sending and receiving packets. Typically a single
packet is sent along each input stream at each input timestamp. A packet can
contain any kind of data, such as a single frame of video or a single integer
@@ -13,6 +13,12 @@ nav_order: 6
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## Real-time timestamps
MediaPipe calculator graphs are often used to process streams of video or audio
@@ -13,6 +13,12 @@ nav_order: 4
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## Scheduling mechanics
Data processing in a MediaPipe graph occurs inside processing nodes defined as
+6
View File
@@ -15,6 +15,12 @@ nav_order: 1
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
Please follow instructions below to build Android example apps in the supported
MediaPipe [solutions](../solutions/solutions.md). To learn more about these
example apps, start from [Hello World! on Android](./hello_world_android.md).
@@ -14,6 +14,12 @@ nav_order: 3
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
***Experimental Only***
The MediaPipe Android Archive (AAR) library is a convenient way to use MediaPipe
+4 -8
View File
@@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/
title: MediaPipe Android Solutions
parent: MediaPipe on Android
grand_parent: Getting Started
@@ -13,14 +14,9 @@ nav_order: 2
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023. This content will not be moved to
the new site, but will remain available in the source code repository on an
as-is basis.*
*This notice and web page will be removed on April 3, 2023.*
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
+4 -8
View File
@@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/
title: Building MediaPipe Examples
parent: Getting Started
nav_exclude: true
@@ -12,14 +13,9 @@ nav_exclude: true
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023. This content will not be moved to
the new site, but will remain available in the source code repository on an
as-is basis.*
*This notice and web page will be removed on April 3, 2023.*
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
+6
View File
@@ -15,6 +15,12 @@ nav_order: 5
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
Please follow instructions below to build C++ command-line example apps in the
supported MediaPipe [solutions](../solutions/solutions.md). To learn more about
these example apps, start from [Hello World! in C++](./hello_world_cpp.md).
+9 -3
View File
@@ -13,6 +13,12 @@ nav_order: 9
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
### How to convert ImageFrames and GpuBuffers
The Calculators [`ImageFrameToGpuBufferCalculator`] and
@@ -47,7 +53,7 @@ calculators need only to be *thread-compatible* and not *thread-safe*.
In order to enable one calculator to process multiple inputs in parallel, there
are two possible approaches:
1. Define multiple calulator nodes and dispatch input packets to all nodes.
1. Define multiple calculator nodes and dispatch input packets to all nodes.
2. Make the calculator thread-safe and configure its [`max_in_flight`] setting.
The first approach can be followed using the calculators designed to distribute
@@ -89,12 +95,12 @@ while the application is running:
The first approach has the advantage of leveraging [`CalculatorGraphConfig`]
processing tools such as "subgraphs". The second approach has the advantage of
allowing active calculators and packets to remain in-flight while settings
change. Mediapipe contributors are currently investigating alternative approaches
change. MediaPipe contributors are currently investigating alternative approaches
to achieve both of these advantages.
### How to process realtime input streams
The mediapipe framework can be used to process data streams either online or
The MediaPipe framework can be used to process data streams either online or
offline. For offline processing, packets are pushed into the graph as soon as
calculators are ready to process those packets. For online processing, one
packet for each frame is pushed into the graph as that frame is recorded.
+4 -8
View File
@@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/
title: Getting Started
nav_order: 2
has_children: true
@@ -12,13 +13,8 @@ has_children: true
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023. This content will not be moved to
the new site, but will remain available in the source code repository on an
as-is basis.*
*This notice and web page will be removed on April 3, 2023.*
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
+6
View File
@@ -13,6 +13,12 @@ nav_order: 7
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## OpenGL ES Support
MediaPipe supports OpenGL ES up to version 3.2 on Android/Linux and up to ES 3.0
@@ -14,6 +14,12 @@ nav_order: 1
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## Introduction
This codelab uses MediaPipe on an Android device.
+6
View File
@@ -14,6 +14,12 @@ nav_order: 1
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
1. Ensure you have a working version of MediaPipe. See
[installation instructions](./install.md).
+6
View File
@@ -14,6 +14,12 @@ nav_order: 1
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## Introduction
This codelab uses MediaPipe on an iOS device.
+6
View File
@@ -13,6 +13,12 @@ nav_order: 8
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## Technical questions
For help with technical or algorithmic questions, visit
+7 -1
View File
@@ -13,6 +13,12 @@ nav_order: 6
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
Note: To interoperate with OpenCV, OpenCV 3.x to 4.1 are preferred. OpenCV
2.x currently works but interoperability support may be deprecated in the
future.
@@ -577,7 +583,7 @@ next section.
Option 1. Follow
[the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
to install Bazel 5.2.0 or higher.
to install Bazel 6.1.1 or higher.
Option 2. Follow the official
[Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+10 -4
View File
@@ -15,6 +15,12 @@ nav_order: 2
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
Please follow instructions below to build iOS example apps in the supported
MediaPipe [solutions](../solutions/solutions.md). To learn more about these
example apps, start from, start from
@@ -114,8 +120,8 @@ allows you to make use of automatic provisioning (see later section).
To install applications on an iOS device, you need a provisioning profile. There
are two options:
1. Automatic provisioning. This allows you to build and install an app to your
personal device. The provisining profile is managed by Xcode, and has to be
1. Automatic provisioning. This allows you to build and install an app on your
personal device. The provisioning profile is managed by Xcode, and has to be
updated often (it is valid for about a week).
2. Custom provisioning. This uses a provisioning profile associated with an
@@ -180,7 +186,7 @@ Profiles"`. If there are none, generate and download a profile on
Note: if you had previously set up automatic provisioning, you should remove the
`provisioning_profile.mobileprovision` symlink in each example's directory,
since it will take precedence over the common one. You can also overwrite it
with you own profile if you need a different profile for different apps.
with your own profile if you need a different profile for different apps.
1. Open `mediapipe/examples/ios/bundle_id.bzl`, and change the
`BUNDLE_ID_PREFIX` to a prefix associated with your provisioning profile.
@@ -197,7 +203,7 @@ Note: When you ask Xcode to run an app, by default it will use the Debug
configuration. Some of our demos are computationally heavy; you may want to use
the Release configuration for better performance.
Note: Due to an imcoptibility caused by one of our dependencies, MediaPipe
Note: Due to an incompatibility caused by one of our dependencies, MediaPipe
cannot be used for apps running on the iPhone Simulator on Apple Silicon (M1).
Tip: To switch build configuration in Xcode, click on the target menu, choose
+3 -7
View File
@@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/
title: MediaPipe in JavaScript
parent: Getting Started
nav_order: 4
@@ -14,12 +15,7 @@ nav_order: 4
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023. This content will not be moved to
the new site, but will remain available in the source code repository on an
as-is basis.*
*This notice and web page will be removed on April 3, 2023.*
as the primary developer documentation site for MediaPipe starting April 3, 2023.*
----
+8 -1
View File
@@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/
title: MediaPipe in Python
parent: Getting Started
has_children: true
@@ -14,6 +15,12 @@ nav_order: 3
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## Ready-to-use Python Solutions
MediaPipe offers ready-to-use yet customizable Python solutions as a prebuilt
+7 -2
View File
@@ -12,6 +12,11 @@ nav_order: 1
1. TOC
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
The MediaPipe Python framework grants direct access to the core components of
the MediaPipe C++ framework such as Timestamp, Packet, and CalculatorGraph,
@@ -76,7 +81,7 @@ np.ndarray | mp::Matrix | create_ma
Google Proto Message | Google Proto Message | create_proto(proto) | get_proto(packet)
List\[Proto\] | std::vector\<Proto\> | n/a | get_proto_list(packet)
It's not uncommon that users create custom C++ classes and and send those into
It's not uncommon that users create custom C++ classes and send those into
the graphs and calculators. To allow the custom classes to be used in Python
with MediaPipe, you may extend the Packet API for a new data type in the
following steps:
@@ -229,7 +234,7 @@ three stages: initialization and setup, graph run, and graph shutdown.
output_packets.append(mp.packet_getter.get_str(packet)))
```
Option 2. Initialize a CalculatorGraph with with a binary protobuf file, and
Option 2. Initialize a CalculatorGraph with a binary protobuf file, and
observe the output stream(s).
```python
+6
View File
@@ -13,6 +13,12 @@ nav_order: 10
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
## Missing Python binary path
The error message:
+14 -9
View File
@@ -6,6 +6,20 @@ nav_order: 1
![MediaPipe](https://mediapipe.dev/images/mediapipe_small.png)
----
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
*This notice and web page will be removed on June 1, 2023.*
----
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
--------------------------------------------------------------------------------
## Live ML anywhere
@@ -21,15 +35,6 @@ ML solutions for live and streaming media.
----
**Attention:** *Thanks for your interest in MediaPipe! We are moving to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation
site for MediaPipe starting April 3, 2023.*
*This notice and web page will be removed on April 3, 2023.*
----
## ML solutions in MediaPipe
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
+1 -1
View File
@@ -1,3 +1,3 @@
MediaPipe
=====================================
Please see https://docs.mediapipe.dev.
Please see https://developers.google.com/mediapipe/
+4 -5
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: AutoFlip (Saliency-aware Video Cropping)
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 14
---
@@ -20,12 +21,10 @@ nav_order: 14
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+4 -5
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: Box Tracking
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 10
---
@@ -20,12 +21,10 @@ nav_order: 10
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+4 -5
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/face_detector/
title: Face Detection
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 1
---
@@ -20,12 +21,10 @@ nav_order: 1
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+4 -5
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/face_landmarker/
title: Face Mesh
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 2
---
@@ -20,12 +21,10 @@ nav_order: 2
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+6 -7
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/image_segmenter/
title: Hair Segmentation
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 8
---
@@ -19,13 +20,11 @@ nav_order: 8
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
As of April 4, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/image_segmenter/)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
![hair_segmentation_android_gpu_gif](https://mediapipe.dev/images/mobile/hair_segmentation_android_gpu.gif)
+6 -7
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/hand_landmarker
title: Hands
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 4
---
@@ -19,13 +20,11 @@ nav_order: 4
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
As of March 1, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/hand_landmarker)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+5 -6
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://github.com/google/mediapipe/blob/master/docs/solutions/holistic.md
title: Holistic
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 6
---
@@ -20,12 +21,10 @@ nav_order: 6
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
@@ -76,7 +75,7 @@ previous frame as a guide to the object region on the current one. However,
during fast movements, the tracker can lose the target, which requires the
detector to re-localize it in the image. MediaPipe Holistic uses
[pose](./pose.md) prediction (on every frame) as an additional ROI prior to
reduce the response time of the pipeline when reacting to fast movements. This
reducing the response time of the pipeline when reacting to fast movements. This
also enables the model to retain semantic consistency across the body and its
parts by preventing a mixup between left and right hands or body parts of one
person in the frame with another.
+4 -5
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: Instant Motion Tracking
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 11
---
@@ -20,12 +21,10 @@ nav_order: 11
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+4 -5
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/face_landmarker/
title: Iris
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 3
---
@@ -20,12 +21,10 @@ nav_order: 3
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+4 -5
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: KNIFT (Template-based Feature Matching)
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 13
---
@@ -20,12 +21,10 @@ nav_order: 13
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+3 -4
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: Dataset Preparation with MediaSequence
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 15
---
@@ -24,8 +25,6 @@ For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+5 -4
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: Models and Model Cards
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 30
---
@@ -22,8 +23,6 @@ MediaPipe Legacy Solutions will continue to be provided on an as-is basis.
We encourage you to check out the new MediaPipe Solutions at:
[https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
*This notice and web page will be removed on April 3, 2023.*
----
### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
@@ -108,6 +107,8 @@ one over the other.
* [TFLite model](https://storage.googleapis.com/mediapipe-assets/ssdlite_object_detection.tflite)
* [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
* [TensorFlow model](https://storage.googleapis.com/mediapipe-assets/object_detection_saved_model/archive.zip)
* [Model information](https://storage.googleapis.com/mediapipe-assets/object_detection_saved_model/README.md)
### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
+8 -9
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/object_detector/
title: Object Detection
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 9
---
@@ -19,13 +20,11 @@ nav_order: 9
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
As of March 1, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/object_detector/)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
![object_detection_android_gpu.gif](https://mediapipe.dev/images/mobile/object_detection_android_gpu.gif)
@@ -118,9 +117,9 @@ on how to build MediaPipe examples.
* With a TensorFlow Model
This uses the
[TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
[TensorFlow model](https://storage.googleapis.com/mediapipe-assets/object_detection_saved_model/archive.zip)
( see also
[model info](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)),
[model info](https://storage.googleapis.com/mediapipe-assets/object_detection_saved_model/README.md)),
and the pipeline is implemented in this
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt).
@@ -0,0 +1,89 @@
---
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/object_detector
title: Object Detection
parent: MediaPipe Legacy Solutions
nav_order: 9
---
# MediaPipe Object Detection
{: .no_toc }
<details close markdown="block">
<summary>
Table of contents
</summary>
{: .text-delta }
1. TOC
{:toc}
</details>
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/object_detector)
site.*
----
### TensorFlow model
The model is trained on [MSCOCO 2014](http://cocodataset.org) dataset using [TensorFlow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection). It is a MobileNetV2-based SSD model with 0.5 depth multiplier. Detailed training configuration is in the provided `pipeline.config`. The model is a relatively compact model which has `0.171 mAP` to achieve real-time performance on mobile devices. You can compare it with other models from the [TensorFlow detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf1_detection_zoo.md).
### TFLite model
The TFLite model is converted from the TensorFlow above. The steps needed to convert the model are similar to [this tutorial](https://medium.com/tensorflow/training-and-serving-a-realtime-mobile-object-detector-in-30-minutes-with-cloud-tpus-b78971cf1193) with minor modifications. Assuming now we have a trained TensorFlow model which includes the checkpoint files and the training configuration file, for example the files provided in this repo:
* `model.ckpt.index`
* `model.ckpt.meta`
* `model.ckpt.data-00000-of-00001`
* `pipeline.config`
Make sure you have installed these [python libraries](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf1.md). Then to get the frozen graph, run the `export_tflite_ssd_graph.py` script from the `models/research` directory with this command:
```bash
$ PATH_TO_MODEL=path/to/the/model
$ bazel run object_detection:export_tflite_ssd_graph -- \
--pipeline_config_path ${PATH_TO_MODEL}/pipeline.config \
--trained_checkpoint_prefix ${PATH_TO_MODEL}/model.ckpt \
--output_directory ${PATH_TO_MODEL} \
--add_postprocessing_op=False
```
The exported model contains two files:
* `tflite_graph.pb`
* `tflite_graph.pbtxt`
The difference between this step and the one in [the tutorial](https://medium.com/tensorflow/training-and-serving-a-realtime-mobile-object-detector-in-30-minutes-with-cloud-tpus-b78971cf1193) is that we set `add_postprocessing_op` to False. In MediaPipe, we have provided all the calculators needed for post-processing such that we can exclude the custom TFLite ops for post-processing in the original graph, e.g., non-maximum suppression. This enables the flexibility to integrate with different post-processing algorithms and implementations.
Optional: You can install and use the [graph tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/graph_transforms) to inspect the input/output of the exported model:
```bash
$ bazel run graph_transforms:summarize_graph -- \
--in_graph=${PATH_TO_MODEL}/tflite_graph.pb
```
You should be able to see the input image size of the model is 320x320 and the outputs of the model are:
* `raw_outputs/box_encodings`
* `raw_outputs/class_predictions`
The last step is to convert the model to TFLite. You can look at [this guide](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/r1/convert/cmdline_examples.md) for more detail. For this example, you just need to run:
```bash
$ tflite_convert -- \
--graph_def_file=${PATH_TO_MODEL}/tflite_graph.pb \
--output_file=${PATH_TO_MODEL}/model.tflite \
--input_format=TENSORFLOW_GRAPHDEF \
--output_format=TFLITE \
--inference_type=FLOAT \
--input_shapes=1,320,320,3 \
--input_arrays=normalized_input_image_tensor \
--output_arrays=raw_outputs/box_encodings,raw_outputs/class_predictions
```
Now you have the TFLite model `model.tflite` ready to use with MediaPipe Object Detection graphs. Please see the examples for more detail.
+4 -5
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: Objectron (3D Object Detection)
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 12
---
@@ -20,12 +21,10 @@ nav_order: 12
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+7 -7
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/
title: Pose
parent: Solutions
parent: MediaPipe Legacy Solutions
has_children: true
has_toc: false
nav_order: 5
@@ -22,12 +23,10 @@ nav_order: 5
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
@@ -144,7 +143,7 @@ The landmark model in MediaPipe Pose predicts the location of 33 pose landmarks
:----------------------------------------------------------------------------------------------: |
*Fig 4. 33 pose landmarks.* |
Optionally, MediaPipe Pose can predicts a full-body
Optionally, MediaPipe Pose can predict a full-body
[segmentation mask](#segmentation_mask) represented as a two-class segmentation
(human or background).
@@ -269,6 +268,7 @@ Supported configuration options:
```python
import cv2
import mediapipe as mp
import numpy as np
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_pose = mp.solutions.pose
+5 -6
View File
@@ -1,8 +1,9 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/
title: Pose Classification
parent: Pose
grand_parent: Solutions
grand_parent: MediaPipe Legacy Solutions
nav_order: 1
---
@@ -21,12 +22,10 @@ nav_order: 1
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+6 -7
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/vision/image_segmenter/
title: Selfie Segmentation
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 7
---
@@ -19,13 +20,11 @@ nav_order: 7
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
Solution. For more information, see the new
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
As of April 4, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/image_segmenter/)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
## Overview
+8 -2
View File
@@ -1,12 +1,12 @@
---
layout: default
title: Solutions
title: MediaPipe Legacy Solutions
nav_order: 3
has_children: true
has_toc: false
---
# Solutions
# MediaPipe Legacy Solutions
{: .no_toc }
1. TOC
@@ -29,6 +29,12 @@ Solutions at:
----
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
----
MediaPipe offers open source cross-platform, customizable ML solutions for live
and streaming media.
+4 -5
View File
@@ -1,7 +1,8 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: YouTube-8M Feature Extraction and Model Inference
parent: Solutions
parent: MediaPipe Legacy Solutions
nav_order: 16
---
@@ -20,12 +21,10 @@ nav_order: 16
**Attention:** *Thank you for your interest in MediaPipe Solutions.
We have ended support for this MediaPipe Legacy Solution as of March 1, 2023.
For more information, see the new
For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
site.*
*This notice and web page will be removed on April 3, 2023.*
----
MediaPipe is a useful and general framework for media processing that can assist
+8 -1
View File
@@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/
title: Performance Benchmarking
parent: Tools
nav_order: 3
@@ -12,6 +13,12 @@ nav_order: 3
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
---
*Coming soon.*
Future mediapipe releases will include tools for visualizing and analysing the
+8 -1
View File
@@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/
title: Tools
nav_order: 4
has_children: true
@@ -11,3 +12,9 @@ has_children: true
1. TOC
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
+8 -1
View File
@@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/
title: Tracing and Profiling
parent: Tools
nav_order: 2
@@ -12,6 +13,12 @@ nav_order: 2
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
----
The MediaPipe framework includes a built-in tracer and profiler. The tracer
records various timing events related to packet processing, including the start
and end time of each Calculator::Process call. The tracer writes trace log files
+6
View File
@@ -13,6 +13,12 @@ nav_order: 1
{:toc}
---
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
---
To help users understand the structure of their calculator graphs and to
understand the overall behavior of their machine learning inference pipelines,
we have built the [MediaPipe Visualizer](https://viz.mediapipe.dev/)
@@ -26,10 +26,11 @@
namespace mediapipe {
namespace {
static bool SafeMultiply(int x, int y, int* result) {
static_assert(sizeof(int64) >= 2 * sizeof(int),
static_assert(sizeof(int64_t) >= 2 * sizeof(int),
"Unable to detect overflow after multiplication");
const int64 big = static_cast<int64>(x) * static_cast<int64>(y);
if (big > static_cast<int64>(INT_MIN) && big < static_cast<int64>(INT_MAX)) {
const int64_t big = static_cast<int64_t>(x) * static_cast<int64_t>(y);
if (big > static_cast<int64_t>(INT_MIN) &&
big < static_cast<int64_t>(INT_MAX)) {
if (result != nullptr) *result = static_cast<int>(big);
return true;
} else {
@@ -182,12 +182,12 @@ class SpectrogramCalculator : public CalculatorBase {
int frame_duration_samples_;
int frame_overlap_samples_;
// How many samples we've been passed, used for checking input time stamps.
int64 cumulative_input_samples_;
int64_t cumulative_input_samples_;
// How many frames we've emitted, used for calculating output time stamps.
int64 cumulative_completed_frames_;
int64_t cumulative_completed_frames_;
// How many frames were emitted last, used for estimating the timestamp on
// Close when use_local_timestamp_ is true;
int64 last_completed_frames_;
int64_t last_completed_frames_;
Timestamp initial_input_timestamp_;
int num_input_channels_;
// How many frequency bins we emit (=N_FFT/2 + 1).
@@ -92,8 +92,8 @@ class SpectrogramCalculatorTest
.cos()
.transpose();
}
int64 input_timestamp = round(packet_start_time_seconds *
Timestamp::kTimestampUnitsPerSecond);
int64_t input_timestamp = round(packet_start_time_seconds *
Timestamp::kTimestampUnitsPerSecond);
AppendInputPacket(packet_data, input_timestamp);
total_num_input_samples += packet_size_samples;
}
@@ -116,8 +116,8 @@ class SpectrogramCalculatorTest
double packet_start_time_seconds =
kInitialTimestampOffsetMicroseconds * 1e-6 +
total_num_input_samples / input_sample_rate_;
int64 input_timestamp = round(packet_start_time_seconds *
Timestamp::kTimestampUnitsPerSecond);
int64_t input_timestamp = round(packet_start_time_seconds *
Timestamp::kTimestampUnitsPerSecond);
std::unique_ptr<Matrix> impulse(
new Matrix(Matrix::Zero(1, packet_sizes_samples[i])));
(*impulse)(0, impulse_offsets_samples[i]) = 1.0;
@@ -157,8 +157,8 @@ class SpectrogramCalculatorTest
.cos()
.transpose();
}
int64 input_timestamp = round(packet_start_time_seconds *
Timestamp::kTimestampUnitsPerSecond);
int64_t input_timestamp = round(packet_start_time_seconds *
Timestamp::kTimestampUnitsPerSecond);
AppendInputPacket(packet_data, input_timestamp);
total_num_input_samples += packet_size_samples;
}
@@ -218,7 +218,7 @@ class SpectrogramCalculatorTest
const double expected_timestamp_seconds =
packet_timestamp_offset_seconds +
cumulative_output_frames * frame_step_seconds;
const int64 expected_timestamp_ticks =
const int64_t expected_timestamp_ticks =
expected_timestamp_seconds * Timestamp::kTimestampUnitsPerSecond;
EXPECT_EQ(expected_timestamp_ticks, packet.Timestamp().Value());
// Accept the timestamp of the first packet as the baseline for checking
@@ -54,7 +54,8 @@ TEST_F(StabilizedLogCalculatorTest, BasicOperation) {
std::vector<Matrix> input_data_matrices;
for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
const int64_t timestamp =
input_packet * Timestamp::kTimestampUnitsPerSecond;
Matrix input_data_matrix =
Matrix::Random(kNumChannels, kNumSamples).array().abs();
input_data_matrices.push_back(input_data_matrix);
@@ -80,7 +81,8 @@ TEST_F(StabilizedLogCalculatorTest, OutputScaleWorks) {
std::vector<Matrix> input_data_matrices;
for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
const int64_t timestamp =
input_packet * Timestamp::kTimestampUnitsPerSecond;
Matrix input_data_matrix =
Matrix::Random(kNumChannels, kNumSamples).array().abs();
input_data_matrices.push_back(input_data_matrix);
@@ -109,7 +109,7 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
// Returns the timestamp of a sample on a base, which is usually the time
// stamp of a packet.
Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
int64 number_of_samples) {
int64_t number_of_samples) {
return timestamp_base + round(number_of_samples / sample_rate_ *
Timestamp::kTimestampUnitsPerSecond);
}
@@ -118,10 +118,10 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
// emitted.
int next_frame_step_samples() const {
// All numbers are in input samples.
const int64 current_output_frame_start = static_cast<int64>(
const int64_t current_output_frame_start = static_cast<int64_t>(
round(cumulative_output_frames_ * average_frame_step_samples_));
CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
const int64 next_output_frame_start = static_cast<int64>(
const int64_t next_output_frame_start = static_cast<int64_t>(
round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
return next_output_frame_start - current_output_frame_start;
}
@@ -134,11 +134,11 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
// emulate_fractional_frame_overlap is true.
double average_frame_step_samples_;
int samples_still_to_drop_;
int64 cumulative_output_frames_;
int64_t cumulative_output_frames_;
// "Completed" samples are samples that are no longer needed because
// the framer has completely stepped past them (taking into account
// any overlap).
int64 cumulative_completed_samples_;
int64_t cumulative_completed_samples_;
Timestamp initial_input_timestamp_;
// The current timestamp is updated along with the incoming packets.
Timestamp current_timestamp_;
@@ -49,7 +49,7 @@ class TimeSeriesFramerCalculatorTest
// Returns a float value with the channel and timestamp separated by
// an order of magnitude, for easy parsing by humans.
float TestValue(int64 timestamp_in_microseconds, int channel) {
float TestValue(int64_t timestamp_in_microseconds, int channel) {
return timestamp_in_microseconds + channel / 10.0;
}
@@ -59,7 +59,7 @@ class TimeSeriesFramerCalculatorTest
auto matrix = new Matrix(num_channels, num_samples);
for (int c = 0; c < num_channels; ++c) {
for (int i = 0; i < num_samples; ++i) {
int64 timestamp = time_series_util::SecondsToSamples(
int64_t timestamp = time_series_util::SecondsToSamples(
starting_timestamp_seconds + i / input_sample_rate_,
Timestamp::kTimestampUnitsPerSecond);
(*matrix)(c, i) = TestValue(timestamp, c);
@@ -429,7 +429,7 @@ class TimeSeriesFramerCalculatorTimestampingTest
num_full_packets -= 1;
}
int64 num_samples = 0;
int64_t num_samples = 0;
for (int packet_num = 0; packet_num < num_full_packets; ++packet_num) {
const Packet& packet = output().packets[packet_num];
num_samples += FrameDurationSamples();
+6 -5
View File
@@ -218,6 +218,7 @@ cc_library(
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
@@ -282,6 +283,7 @@ cc_library(
}),
deps = [
":concatenate_vector_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:classification_cc_proto",
@@ -290,7 +292,6 @@ cc_library(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework:calculator_framework",
"//mediapipe/util:render_data_cc_proto",
"@org_tensorflow//tensorflow/lite:framework",
] + select({
@@ -900,12 +901,12 @@ cc_library(
}),
deps = [
":split_vector_calculator_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
@@ -30,7 +30,7 @@ namespace mediapipe {
typedef ClipVectorSizeCalculator<int> TestClipIntVectorSizeCalculator;
REGISTER_CALCULATOR(TestClipIntVectorSizeCalculator);
void AddInputVector(const std::vector<int>& input, int64 timestamp,
void AddInputVector(const std::vector<int>& input, int64_t timestamp,
CalculatorRunner* runner) {
runner->MutableInputs()->Index(0).packets.push_back(
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
@@ -63,7 +63,7 @@ void ValidateCombinedLandmarks(
void AddInputLandmarkLists(
const std::vector<NormalizedLandmarkList>& input_landmarks_vec,
int64 timestamp, CalculatorRunner* runner) {
int64_t timestamp, CalculatorRunner* runner) {
for (int i = 0; i < input_landmarks_vec.size(); ++i) {
runner->MutableInputs()->Index(i).packets.push_back(
MakePacket<NormalizedLandmarkList>(input_landmarks_vec[i])
@@ -73,7 +73,7 @@ void AddInputLandmarkLists(
void AddInputClassificationLists(
const std::vector<ClassificationList>& input_classifications_vec,
int64 timestamp, CalculatorRunner* runner) {
int64_t timestamp, CalculatorRunner* runner) {
for (int i = 0; i < input_classifications_vec.size(); ++i) {
runner->MutableInputs()->Index(i).packets.push_back(
MakePacket<ClassificationList>(input_classifications_vec[i])
@@ -46,10 +46,10 @@ MEDIAPIPE_REGISTER_NODE(ConcatenateFloatVectorCalculator);
// input_stream: "int32_vector_2"
// output_stream: "concatenated_int32_vector"
// }
typedef ConcatenateVectorCalculator<int32> ConcatenateInt32VectorCalculator;
typedef ConcatenateVectorCalculator<int32_t> ConcatenateInt32VectorCalculator;
MEDIAPIPE_REGISTER_NODE(ConcatenateInt32VectorCalculator);
typedef ConcatenateVectorCalculator<uint64> ConcatenateUInt64VectorCalculator;
typedef ConcatenateVectorCalculator<uint64_t> ConcatenateUInt64VectorCalculator;
MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator);
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
@@ -30,26 +30,26 @@ namespace mediapipe {
typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
void AddInputVector(int index, const std::vector<int>& input, int64 timestamp,
void AddInputVector(int index, const std::vector<int>& input, int64_t timestamp,
CalculatorRunner* runner) {
runner->MutableInputs()->Index(index).packets.push_back(
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
}
void AddInputVectors(const std::vector<std::vector<int>>& inputs,
int64 timestamp, CalculatorRunner* runner) {
int64_t timestamp, CalculatorRunner* runner) {
for (int i = 0; i < inputs.size(); ++i) {
AddInputVector(i, inputs[i], timestamp, runner);
}
}
void AddInputItem(int index, int input, int64 timestamp,
void AddInputItem(int index, int input, int64_t timestamp,
CalculatorRunner* runner) {
runner->MutableInputs()->Index(index).packets.push_back(
MakePacket<int>(input).At(Timestamp(timestamp)));
}
void AddInputItems(const std::vector<int>& inputs, int64 timestamp,
void AddInputItems(const std::vector<int>& inputs, int64_t timestamp,
CalculatorRunner* runner) {
for (int i = 0; i < inputs.size(); ++i) {
AddInputItem(i, inputs[i], timestamp, runner);
@@ -279,7 +279,7 @@ TEST(TestConcatenateIntVectorCalculatorTest, MixedVectorsAndItemsAnother) {
}
void AddInputVectors(const std::vector<std::vector<float>>& inputs,
int64 timestamp, CalculatorRunner* runner) {
int64_t timestamp, CalculatorRunner* runner) {
for (int i = 0; i < inputs.size(); ++i) {
runner->MutableInputs()->Index(i).packets.push_back(
MakePacket<std::vector<float>>(inputs[i]).At(Timestamp(timestamp)));
@@ -18,6 +18,7 @@
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
@@ -67,4 +68,8 @@ REGISTER_CALCULATOR(EndLoopMatrixCalculator);
typedef EndLoopCalculator<std::vector<Tensor>> EndLoopTensorCalculator;
REGISTER_CALCULATOR(EndLoopTensorCalculator);
typedef EndLoopCalculator<std::vector<::mediapipe::Image>>
EndLoopImageCalculator;
REGISTER_CALCULATOR(EndLoopImageCalculator);
} // namespace mediapipe
@@ -61,8 +61,8 @@ class AtomicSemaphore {
};
// Returns the timestamp values for a vector of Packets.
std::vector<int64> TimestampValues(const std::vector<Packet>& packets) {
std::vector<int64> result;
std::vector<int64_t> TimestampValues(const std::vector<Packet>& packets) {
std::vector<int64_t> result;
for (const Packet& packet : packets) {
result.push_back(packet.Timestamp().Value());
}
@@ -180,9 +180,9 @@ TEST_F(FlowLimiterCalculatorSemaphoreTest, FramesDropped) {
InitializeGraph(1);
MP_ASSERT_OK(graph_.StartRun({}));
auto send_packet = [this](const std::string& input_name, int64 n) {
auto send_packet = [this](const std::string& input_name, int64_t n) {
MP_EXPECT_OK(graph_.AddPacketToInputStream(
input_name, MakePacket<int64>(n).At(Timestamp(n))));
input_name, MakePacket<int64_t>(n).At(Timestamp(n))));
};
Packet allow_packet;
@@ -207,12 +207,12 @@ TEST_F(FlowLimiterCalculatorSemaphoreTest, FramesDropped) {
EXPECT_EQ(10, out_1_packets_.size());
// Timestamps have not been altered.
EXPECT_EQ(PacketValues<int64>(out_1_packets_),
EXPECT_EQ(PacketValues<int64_t>(out_1_packets_),
TimestampValues(out_1_packets_));
// Extra inputs on in_1 have been dropped.
EXPECT_EQ(TimestampValues(out_1_packets_),
(std::vector<int64>{0, 10, 20, 30, 40, 50, 60, 70, 80, 90}));
(std::vector<int64_t>{0, 10, 20, 30, 40, 50, 60, 70, 80, 90}));
}
// A calculator that sleeps during Process.
@@ -221,8 +221,8 @@ class SleepCalculator : public CalculatorBase {
static absl::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kPacketTag).SetAny();
cc->Outputs().Tag(kPacketTag).SetSameAs(&cc->Inputs().Tag(kPacketTag));
cc->InputSidePackets().Tag(kSleepTimeTag).Set<int64>();
cc->InputSidePackets().Tag(kWarmupTimeTag).Set<int64>();
cc->InputSidePackets().Tag(kSleepTimeTag).Set<int64_t>();
cc->InputSidePackets().Tag(kWarmupTimeTag).Set<int64_t>();
cc->InputSidePackets().Tag(kClockTag).Set<mediapipe::Clock*>();
cc->SetTimestampOffset(0);
return absl::OkStatus();
@@ -237,8 +237,8 @@ class SleepCalculator : public CalculatorBase {
++packet_count;
absl::Duration sleep_time = absl::Microseconds(
packet_count == 1
? cc->InputSidePackets().Tag(kWarmupTimeTag).Get<int64>()
: cc->InputSidePackets().Tag(kSleepTimeTag).Get<int64>());
? cc->InputSidePackets().Tag(kWarmupTimeTag).Get<int64_t>()
: cc->InputSidePackets().Tag(kSleepTimeTag).Get<int64_t>());
clock_->Sleep(sleep_time);
cc->Outputs()
.Tag(kPacketTag)
@@ -375,8 +375,8 @@ TEST_F(FlowLimiterCalculatorTest, FinishedTimestamps) {
std::map<std::string, Packet> side_packets = {
{"limiter_options",
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
{"warmup_time", MakePacket<int64>(22000)},
{"sleep_time", MakePacket<int64>(22000)},
{"warmup_time", MakePacket<int64_t>(22000)},
{"sleep_time", MakePacket<int64_t>(22000)},
{"drop_timesamps", MakePacket<bool>(false)},
{"clock", MakePacket<mediapipe::Clock*>(clock_)},
};
@@ -447,8 +447,8 @@ TEST_F(FlowLimiterCalculatorTest, FinishedLost) {
std::map<std::string, Packet> side_packets = {
{"limiter_options",
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
{"warmup_time", MakePacket<int64>(22000)},
{"sleep_time", MakePacket<int64>(22000)},
{"warmup_time", MakePacket<int64_t>(22000)},
{"sleep_time", MakePacket<int64_t>(22000)},
{"drop_timesamps", MakePacket<bool>(true)},
{"clock", MakePacket<mediapipe::Clock*>(clock_)},
};
@@ -511,8 +511,8 @@ TEST_F(FlowLimiterCalculatorTest, FinishedDelayed) {
std::map<std::string, Packet> side_packets = {
{"limiter_options",
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
{"warmup_time", MakePacket<int64>(500000)},
{"sleep_time", MakePacket<int64>(22000)},
{"warmup_time", MakePacket<int64_t>(500000)},
{"sleep_time", MakePacket<int64_t>(22000)},
{"drop_timesamps", MakePacket<bool>(false)},
{"clock", MakePacket<mediapipe::Clock*>(clock_)},
};
@@ -606,8 +606,8 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
std::map<std::string, Packet> side_packets = {
{"limiter_options",
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
{"warmup_time", MakePacket<int64>(22000)},
{"sleep_time", MakePacket<int64>(22000)},
{"warmup_time", MakePacket<int64_t>(22000)},
{"sleep_time", MakePacket<int64_t>(22000)},
{"drop_timesamps", MakePacket<bool>(true)},
{"clock", MakePacket<mediapipe::Clock*>(clock_)},
};
@@ -715,8 +715,8 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
std::map<std::string, Packet> side_packets = {
{"limiter_options",
MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
{"warmup_time", MakePacket<int64>(12000)},
{"sleep_time", MakePacket<int64>(12000)},
{"warmup_time", MakePacket<int64_t>(12000)},
{"sleep_time", MakePacket<int64_t>(12000)},
{"drop_timesamps", MakePacket<bool>(true)},
{"clock", MakePacket<mediapipe::Clock*>(clock_)},
};
@@ -862,9 +862,9 @@ TEST_F(FlowLimiterCalculatorTest, AuxiliaryInputs) {
std::map<std::string, Packet> side_packets = {
// Fake processing lazy initialization time in microseconds.
{"warmup_time", MakePacket<int64>(22000)},
{"warmup_time", MakePacket<int64_t>(22000)},
// Fake processing duration in microseconds.
{"sleep_time", MakePacket<int64>(22000)},
{"sleep_time", MakePacket<int64_t>(22000)},
// The SimulationClock to count virtual elapsed time.
{"clock", MakePacket<mediapipe::Clock*>(clock_)},
};
@@ -125,7 +125,6 @@ class GateCalculator : public CalculatorBase {
RET_CHECK_OK(CheckAndInitAllowDisallowInputs(cc));
const int num_data_streams = cc->Inputs().NumEntries("");
RET_CHECK_GE(num_data_streams, 1);
RET_CHECK_EQ(cc->Outputs().NumEntries(""), num_data_streams)
<< "Number of data output streams must match with data input streams.";
@@ -52,6 +52,15 @@ class GateCalculatorTest : public ::testing::Test {
MP_ASSERT_OK(runner_->Run()) << "Calculator execution failed.";
}
void RunTimeStepWithoutDataStream(int64_t timestamp,
const std::string& control_tag,
bool control) {
runner_->MutableInputs()
->Tag(control_tag)
.packets.push_back(MakePacket<bool>(control).At(Timestamp(timestamp)));
MP_ASSERT_OK(runner_->Run()) << "Calculator execution failed.";
}
void SetRunner(const std::string& proto) {
runner_ = absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(proto));
@@ -332,6 +341,35 @@ TEST_F(GateCalculatorTest, AllowWithStateChange) {
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
}
TEST_F(GateCalculatorTest, AllowWithStateChangeNoDataStreams) {
SetRunner(R"(
calculator: "GateCalculator"
input_stream: "ALLOW:gating_stream"
output_stream: "STATE_CHANGE:state_changed"
)");
constexpr int64_t kTimestampValue0 = 42;
RunTimeStepWithoutDataStream(kTimestampValue0, "ALLOW", false);
constexpr int64_t kTimestampValue1 = 43;
RunTimeStepWithoutDataStream(kTimestampValue1, "ALLOW", true);
constexpr int64_t kTimestampValue2 = 44;
RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true);
constexpr int64_t kTimestampValue3 = 45;
RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false);
LOG(INFO) << "a";
const std::vector<Packet>& output =
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
LOG(INFO) << "s";
ASSERT_EQ(2, output.size());
LOG(INFO) << "d";
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
LOG(INFO) << "f";
EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
LOG(INFO) << "g";
}
TEST_F(GateCalculatorTest, DisallowWithStateChange) {
SetRunner(R"(
calculator: "GateCalculator"
@@ -359,6 +397,31 @@ TEST_F(GateCalculatorTest, DisallowWithStateChange) {
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
}
TEST_F(GateCalculatorTest, DisallowWithStateChangeNoDataStreams) {
SetRunner(R"(
calculator: "GateCalculator"
input_stream: "DISALLOW:gating_stream"
output_stream: "STATE_CHANGE:state_changed"
)");
constexpr int64_t kTimestampValue0 = 42;
RunTimeStepWithoutDataStream(kTimestampValue0, "DISALLOW", true);
constexpr int64_t kTimestampValue1 = 43;
RunTimeStepWithoutDataStream(kTimestampValue1, "DISALLOW", false);
constexpr int64_t kTimestampValue2 = 44;
RunTimeStepWithoutDataStream(kTimestampValue2, "DISALLOW", false);
constexpr int64_t kTimestampValue3 = 45;
RunTimeStepWithoutDataStream(kTimestampValue3, "DISALLOW", true);
const std::vector<Packet>& output =
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
ASSERT_EQ(2, output.size());
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
}
// Must not detect disallow value for first timestamp as a state change.
TEST_F(GateCalculatorTest, DisallowInitialNoStateTransition) {
SetRunner(R"(
@@ -94,17 +94,17 @@ class GraphProfileCalculatorTest : public ::testing::Test {
&graph_config_));
}
static Packet PacketAt(int64 ts) {
return Adopt(new int64(999)).At(Timestamp(ts));
static Packet PacketAt(int64_t ts) {
return Adopt(new int64_t(999)).At(Timestamp(ts));
}
static Packet None() { return Packet().At(Timestamp::OneOverPostStream()); }
static bool IsNone(const Packet& packet) {
return packet.Timestamp() == Timestamp::OneOverPostStream();
}
// Return the values of the timestamps of a vector of Packets.
static std::vector<int64> TimestampValues(
static std::vector<int64_t> TimestampValues(
const std::vector<Packet>& packets) {
std::vector<int64> result;
std::vector<int64_t> result;
for (const Packet& p : packets) {
result.push_back(p.Timestamp().Value());
}
@@ -191,17 +191,17 @@ class ImmediateMuxCalculatorTest : public ::testing::Test {
&graph_config_));
}
static Packet PacketAt(int64 ts) {
return Adopt(new int64(999)).At(Timestamp(ts));
static Packet PacketAt(int64_t ts) {
return Adopt(new int64_t(999)).At(Timestamp(ts));
}
static Packet None() { return Packet().At(Timestamp::OneOverPostStream()); }
static bool IsNone(const Packet& packet) {
return packet.Timestamp() == Timestamp::OneOverPostStream();
}
// Return the values of the timestamps of a vector of Packets.
static std::vector<int64> TimestampValues(
static std::vector<int64_t> TimestampValues(
const std::vector<Packet>& packets) {
std::vector<int64> result;
std::vector<int64_t> result;
for (const Packet& p : packets) {
result.push_back(p.Timestamp().Value());
}
@@ -61,8 +61,8 @@ class SimpleRunner : public CalculatorRunner {
MutableInputs()->Index(0).header = Adopt(video_header.release());
}
std::vector<int64> GetOutputTimestamps() const {
std::vector<int64> timestamps;
std::vector<int64_t> GetOutputTimestamps() const {
std::vector<int64_t> timestamps;
for (const Packet& packet : Outputs().Index(0).packets) {
timestamps.emplace_back(packet.Timestamp().Value());
}
@@ -90,7 +90,7 @@ TEST(PacketThinnerCalculatorTest, StartAndEndTimeTest) {
runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {5, 11};
const std::vector<int64_t> expected_timestamps = {5, 11};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -104,7 +104,7 @@ TEST(PacketThinnerCalculatorTest, AsyncUniformStreamThinningTest) {
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {2, 8, 14};
const std::vector<int64_t> expected_timestamps = {2, 8, 14};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -123,10 +123,10 @@ TEST(PacketThinnerCalculatorTest, ASyncUniformStreamThinningTestBySidePacket) {
SimpleRunner runner(node);
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
runner.MutableSidePackets()->Tag(kPeriodTag) = MakePacket<int64>(5);
runner.MutableSidePackets()->Tag(kPeriodTag) = MakePacket<int64_t>(5);
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {2, 8, 14};
const std::vector<int64_t> expected_timestamps = {2, 8, 14};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -143,7 +143,7 @@ TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest1) {
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
const std::vector<int64_t> expected_timestamps = {2, 6, 10, 14};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -162,10 +162,10 @@ TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTestBySidePacket1) {
SimpleRunner runner(node);
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
runner.MutableSidePackets()->Tag(kPeriodTag) = MakePacket<int64>(5);
runner.MutableSidePackets()->Tag(kPeriodTag) = MakePacket<int64_t>(5);
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
const std::vector<int64_t> expected_timestamps = {2, 6, 10, 14};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -182,7 +182,7 @@ TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest2) {
runner.SetInput({2, 4, 6, 8, 10, 12, 14});
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {0, 5, 10, 15};
const std::vector<int64_t> expected_timestamps = {0, 5, 10, 15};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -200,7 +200,7 @@ TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest1) {
runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {2, 7, 13, 19, 29};
const std::vector<int64_t> expected_timestamps = {2, 7, 13, 19, 29};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -217,7 +217,7 @@ TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest2) {
runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {2, 5, 11, 17, 19, 23, 29};
const std::vector<int64_t> expected_timestamps = {2, 5, 11, 17, 19, 23, 29};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -236,7 +236,7 @@ TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest1) {
runner.SetInput({2, 3});
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {0, 5};
const std::vector<int64_t> expected_timestamps = {0, 5};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -254,7 +254,7 @@ TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest2) {
runner.SetInput({-4, -3, 8, 9});
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {-6, 0, 6, 12};
const std::vector<int64_t> expected_timestamps = {-6, 0, 6, 12};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
}
@@ -270,7 +270,7 @@ TEST(PacketThinnerCalculatorTest, FrameRateTest1) {
runner.SetFrameRate(1000000.0 / 2);
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {2, 8, 14};
const std::vector<int64_t> expected_timestamps = {2, 8, 14};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
// The true sampling period is 6.
EXPECT_DOUBLE_EQ(1000000.0 / 6, runner.GetFrameRate());
@@ -287,7 +287,7 @@ TEST(PacketThinnerCalculatorTest, FrameRateTest2) {
runner.SetInput({8, 16, 24, 32, 40, 48, 56});
runner.SetFrameRate(1000000.0 / 8);
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {8, 16, 24, 32, 40, 48, 56};
const std::vector<int64_t> expected_timestamps = {8, 16, 24, 32, 40, 48, 56};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
// The true sampling period is still 8.
EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate());
@@ -308,7 +308,7 @@ TEST(PacketThinnerCalculatorTest, FrameRateTest3) {
runner.SetFrameRate(1000000.0 / 2);
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
const std::vector<int64_t> expected_timestamps = {2, 6, 10, 14};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
// The true (long-run) sampling period is 5.
EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate());
@@ -329,7 +329,7 @@ TEST(PacketThinnerCalculatorTest, FrameRateTest4) {
runner.SetFrameRate(1000000.0 / 2);
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {0, 5, 10, 15};
const std::vector<int64_t> expected_timestamps = {0, 5, 10, 15};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
// The true (long-run) sampling period is 5.
EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate());
@@ -349,7 +349,7 @@ TEST(PacketThinnerCalculatorTest, FrameRateTest5) {
runner.SetFrameRate(1000000.0 / 8);
MP_ASSERT_OK(runner.Run());
const std::vector<int64> expected_timestamps = {10, 15, 25, 30, 40, 50, 55};
const std::vector<int64_t> expected_timestamps = {10, 15, 25, 30, 40, 50, 55};
EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
// The true (long-run) sampling period is 8.
EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate());
@@ -52,8 +52,8 @@ class AtomicSemaphore {
};
// Returns the timestamp values for a vector of Packets.
std::vector<int64> TimestampValues(const std::vector<Packet>& packets) {
std::vector<int64> result;
std::vector<int64_t> TimestampValues(const std::vector<Packet>& packets) {
std::vector<int64_t> result;
for (const Packet& packet : packets) {
result.push_back(packet.Timestamp().Value());
}
@@ -283,9 +283,9 @@ TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) {
InitializeGraph(1);
MP_ASSERT_OK(graph_.StartRun({}));
auto send_packet = [this](const std::string& input_name, int64 n) {
auto send_packet = [this](const std::string& input_name, int64_t n) {
MP_EXPECT_OK(graph_.AddPacketToInputStream(
input_name, MakePacket<int64>(n).At(Timestamp(n))));
input_name, MakePacket<int64_t>(n).At(Timestamp(n))));
};
for (int i = 0; i < 10; i++) {
@@ -307,14 +307,14 @@ TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) {
EXPECT_EQ(10, out_2_packets_.size());
// Timestamps have not been messed with.
EXPECT_EQ(PacketValues<int64>(out_1_packets_),
EXPECT_EQ(PacketValues<int64_t>(out_1_packets_),
TimestampValues(out_1_packets_));
EXPECT_EQ(PacketValues<int64>(out_2_packets_),
EXPECT_EQ(PacketValues<int64_t>(out_2_packets_),
TimestampValues(out_2_packets_));
// Extra inputs on in_1 have been dropped
EXPECT_EQ(TimestampValues(out_1_packets_),
(std::vector<int64>{0, 10, 20, 30, 40, 50, 60, 70, 80, 90}));
(std::vector<int64_t>{0, 10, 20, 30, 40, 50, 60, 70, 80, 90}));
EXPECT_EQ(TimestampValues(out_1_packets_), TimestampValues(out_2_packets_));
// The closing of the stream has been propagated.
@@ -339,7 +339,7 @@ TEST_F(RealTimeFlowLimiterCalculatorTest, AllStreamsClose) {
EXPECT_EQ(TimestampValues(out_1_packets_), TimestampValues(out_2_packets_));
EXPECT_EQ(TimestampValues(out_1_packets_),
(std::vector<int64>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}));
(std::vector<int64_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}));
EXPECT_EQ(1, close_count_);
}
@@ -392,50 +392,50 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
send_packet("in_a", 1);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(allow, false);
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{}));
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64_t>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64_t>{}));
send_packet("in_a", 2);
send_packet("in_b", 1);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64_t>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64_t>{1}));
EXPECT_EQ(allow, false);
send_packet("finished", 1);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64_t>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64_t>{1}));
EXPECT_EQ(allow, true);
send_packet("in_b", 2);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64_t>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64_t>{1}));
EXPECT_EQ(allow, true);
send_packet("in_b", 3);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64_t>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64_t>{1, 3}));
EXPECT_EQ(allow, false);
send_packet("in_b", 4);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64_t>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64_t>{1, 3}));
EXPECT_EQ(allow, false);
send_packet("in_a", 3);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64_t>{1, 3}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64_t>{1, 3}));
EXPECT_EQ(allow, false);
send_packet("finished", 3);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64_t>{1, 3}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64_t>{1, 3}));
EXPECT_EQ(allow, true);
MP_EXPECT_OK(graph_.CloseAllInputStreams());
@@ -486,7 +486,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
send_packet("in", 1);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(allow, false);
EXPECT_EQ(TimestampValues(in_sampled_packets_), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(in_sampled_packets_), (std::vector<int64_t>{1}));
MP_EXPECT_OK(in_sampled_packets_[0].Consume<int>());
@@ -322,10 +322,10 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 20;
const int64 expected_timestamp = 5;
const int64_t expected_timestamp = 5;
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)},
{"timestamp", MakePacket<int64>(expected_timestamp)}}));
{"timestamp", MakePacket<int64_t>(expected_timestamp)}}));
MP_ASSERT_OK(graph.WaitUntilDone());
@@ -360,11 +360,11 @@ TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value0 = 20;
const int expected_value1 = 15;
const int64 expected_timestamp = 5;
const int64_t expected_timestamp = 5;
MP_ASSERT_OK(
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
{"side_packet1", MakePacket<int>(expected_value1)},
{"timestamp", MakePacket<int64>(expected_timestamp)}}));
{"timestamp", MakePacket<int64_t>(expected_timestamp)}}));
MP_ASSERT_OK(graph.WaitUntilDone());
@@ -156,9 +156,9 @@ class SplitListsCalculator : public CalculatorBase {
virtual ItemType* AddItem(ListType& list) const = 0;
private:
std::vector<std::pair<int32, int32>> ranges_;
int32 max_range_end_ = -1;
int32 total_elements_ = 0;
std::vector<std::pair<int32_t, int32_t>> ranges_;
int32_t max_range_end_ = -1;
int32_t total_elements_ = 0;
bool element_only_ = false;
bool combine_outputs_ = false;
};
+69 -48
View File
@@ -147,36 +147,9 @@ cc_library(
srcs = ["set_alpha_calculator.cc"],
deps = [
":set_alpha_calculator_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:shader_util",
],
}),
alwayslink = 1,
)
cc_library(
name = "bilateral_filter_calculator",
srcs = ["bilateral_filter_calculator.cc"],
deps = [
":bilateral_filter_calculator_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework:calculator_options_cc_proto",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:logging",
@@ -188,8 +161,55 @@ cc_library(
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
],
}),
alwayslink = 1,
)
cc_test(
name = "set_alpha_calculator_test",
srcs = ["set_alpha_calculator_test.cc"],
deps = [
":set_alpha_calculator",
":set_alpha_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "bilateral_filter_calculator",
srcs = ["bilateral_filter_calculator.cc"],
deps = [
":bilateral_filter_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
"@com_google_absl//absl/strings",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
],
}),
@@ -229,12 +249,11 @@ cc_library(
"//conditions:default": [],
}),
deps = [
":rotation_mode_cc_proto",
":image_transformation_calculator_cc_proto",
":rotation_mode_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework:timestamp",
"//mediapipe/gpu:scale_mode_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:video_stream_header",
@@ -242,12 +261,13 @@ cc_library(
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:scale_mode_cc_proto",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
],
}),
@@ -273,10 +293,10 @@ cc_library(
}),
deps = [
":image_cropping_calculator_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
@@ -285,8 +305,8 @@ cc_library(
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:shader_util",
],
@@ -347,20 +367,20 @@ cc_library(
srcs = ["recolor_calculator.cc"],
deps = [
":recolor_calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/util:color_cc_proto",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
],
}),
@@ -420,8 +440,8 @@ cc_library(
srcs = ["image_clone_calculator.cc"],
deps = [
":image_clone_calculator_cc_proto",
"//mediapipe/framework/api2:node",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
@@ -438,8 +458,8 @@ cc_library(
name = "image_properties_calculator",
srcs = ["image_properties_calculator.cc"],
deps = [
"//mediapipe/framework/api2:node",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
@@ -600,11 +620,11 @@ cc_library(
srcs = ["segmentation_smoothing_calculator.cc"],
deps = [
":segmentation_smoothing_calculator_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
@@ -612,8 +632,8 @@ cc_library(
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
],
}) + select({
@@ -708,8 +728,6 @@ cc_library(
deps = [
":affine_transformation",
":warp_affine_calculator_cc_proto",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/api2:port",
@@ -717,12 +735,14 @@ cc_library(
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
":affine_transformation_runner_gl",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gpu_buffer",
":affine_transformation_runner_gl",
],
}) + select({
"//mediapipe/framework/port:disable_opencv": [],
@@ -748,6 +768,7 @@ cc_test(
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation_border_zero.png",
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation.png",
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero_interp_cubic.png",
"//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
],
tags = ["desktop_only_test"],
@@ -29,6 +29,9 @@ class AffineTransformation {
// pixels will be calculated.
enum class BorderMode { kZero, kReplicate };
// Pixel sampling interpolation method.
enum class Interpolation { kLinear, kCubic };
struct Size {
int width;
int height;
@@ -77,8 +77,11 @@ class GlTextureWarpAffineRunner
std::unique_ptr<GpuBuffer>> {
public:
GlTextureWarpAffineRunner(std::shared_ptr<GlCalculatorHelper> gl_helper,
GpuOrigin::Mode gpu_origin)
: gl_helper_(gl_helper), gpu_origin_(gpu_origin) {}
GpuOrigin::Mode gpu_origin,
AffineTransformation::Interpolation interpolation)
: gl_helper_(gl_helper),
gpu_origin_(gpu_origin),
interpolation_(interpolation) {}
absl::Status Init() {
return gl_helper_->RunInGlContext([this]() -> absl::Status {
const GLint attr_location[kNumAttributes] = {
@@ -103,28 +106,83 @@ class GlTextureWarpAffineRunner
}
)";
// TODO Move bicubic code to common shared place.
constexpr GLchar kFragShader[] = R"(
DEFAULT_PRECISION(highp, float)
in vec2 sample_coordinate;
uniform sampler2D input_texture;
DEFAULT_PRECISION(highp, float)
#ifdef GL_ES
#define fragColor gl_FragColor
#else
out vec4 fragColor;
#endif // defined(GL_ES);
in vec2 sample_coordinate;
uniform sampler2D input_texture;
uniform vec2 input_size;
void main() {
vec4 color = texture2D(input_texture, sample_coordinate);
#ifdef CUSTOM_ZERO_BORDER_MODE
float out_of_bounds =
float(sample_coordinate.x < 0.0 || sample_coordinate.x > 1.0 ||
sample_coordinate.y < 0.0 || sample_coordinate.y > 1.0);
color = mix(color, vec4(0.0, 0.0, 0.0, 0.0), out_of_bounds);
#endif // defined(CUSTOM_ZERO_BORDER_MODE)
fragColor = color;
}
)";
#ifdef GL_ES
#define fragColor gl_FragColor
#else
out vec4 fragColor;
#endif // defined(GL_ES);
#ifdef CUBIC_INTERPOLATION
vec4 sample(sampler2D tex, vec2 tex_coord, vec2 tex_size) {
const vec2 halve = vec2(0.5,0.5);
const vec2 one = vec2(1.0,1.0);
const vec2 two = vec2(2.0,2.0);
const vec2 three = vec2(3.0,3.0);
const vec2 six = vec2(6.0,6.0);
// Calculate the fraction and integer.
tex_coord = tex_coord * tex_size - halve;
vec2 frac = fract(tex_coord);
vec2 index = tex_coord - frac + halve;
// Calculate weights for Catmull-Rom filter.
vec2 w0 = frac * (-halve + frac * (one - halve * frac));
vec2 w1 = one + frac * frac * (-(two+halve) + three/two * frac);
vec2 w2 = frac * (halve + frac * (two - three/two * frac));
vec2 w3 = frac * frac * (-halve + halve * frac);
// Calculate weights to take advantage of bilinear texture lookup.
vec2 w12 = w1 + w2;
vec2 offset12 = w2 / (w1 + w2);
vec2 index_tl = index - one;
vec2 index_br = index + two;
vec2 index_eq = index + offset12;
index_tl /= tex_size;
index_br /= tex_size;
index_eq /= tex_size;
// 9 texture lookup and linear blending.
vec4 color = vec4(0.0);
color += texture2D(tex, vec2(index_tl.x, index_tl.y)) * w0.x * w0.y;
color += texture2D(tex, vec2(index_eq.x, index_tl.y)) * w12.x *w0.y;
color += texture2D(tex, vec2(index_br.x, index_tl.y)) * w3.x * w0.y;
color += texture2D(tex, vec2(index_tl.x, index_eq.y)) * w0.x * w12.y;
color += texture2D(tex, vec2(index_eq.x, index_eq.y)) * w12.x *w12.y;
color += texture2D(tex, vec2(index_br.x, index_eq.y)) * w3.x * w12.y;
color += texture2D(tex, vec2(index_tl.x, index_br.y)) * w0.x * w3.y;
color += texture2D(tex, vec2(index_eq.x, index_br.y)) * w12.x *w3.y;
color += texture2D(tex, vec2(index_br.x, index_br.y)) * w3.x * w3.y;
return color;
}
#else
vec4 sample(sampler2D tex, vec2 tex_coord, vec2 tex_size) {
return texture2D(tex, tex_coord);
}
#endif // defined(CUBIC_INTERPOLATION)
void main() {
vec4 color = sample(input_texture, sample_coordinate, input_size);
#ifdef CUSTOM_ZERO_BORDER_MODE
float out_of_bounds =
float(sample_coordinate.x < 0.0 || sample_coordinate.x > 1.0 ||
sample_coordinate.y < 0.0 || sample_coordinate.y > 1.0);
color = mix(color, vec4(0.0, 0.0, 0.0, 0.0), out_of_bounds);
#endif // defined(CUSTOM_ZERO_BORDER_MODE)
fragColor = color;
}
)";
// Create program and set parameters.
auto create_fn = [&](const std::string& vs,
@@ -137,14 +195,28 @@ class GlTextureWarpAffineRunner
glUseProgram(program);
glUniform1i(glGetUniformLocation(program, "input_texture"), 1);
GLint matrix_id = glGetUniformLocation(program, "transform_matrix");
return Program{.id = program, .matrix_id = matrix_id};
GLint size_id = glGetUniformLocation(program, "input_size");
return Program{
.id = program, .matrix_id = matrix_id, .size_id = size_id};
};
const std::string vert_src =
absl::StrCat(mediapipe::kMediaPipeVertexShaderPreamble, kVertShader);
const std::string frag_src = absl::StrCat(
mediapipe::kMediaPipeFragmentShaderPreamble, kFragShader);
std::string interpolation_def;
switch (interpolation_) {
case AffineTransformation::Interpolation::kCubic:
interpolation_def = R"(
#define CUBIC_INTERPOLATION
)";
break;
case AffineTransformation::Interpolation::kLinear:
break;
}
const std::string frag_src =
absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble,
interpolation_def, kFragShader);
ASSIGN_OR_RETURN(program_, create_fn(vert_src, frag_src));
@@ -152,9 +224,9 @@ class GlTextureWarpAffineRunner
std::string custom_zero_border_mode_def = R"(
#define CUSTOM_ZERO_BORDER_MODE
)";
const std::string frag_custom_zero_src =
absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble,
custom_zero_border_mode_def, kFragShader);
const std::string frag_custom_zero_src = absl::StrCat(
mediapipe::kMediaPipeFragmentShaderPreamble,
custom_zero_border_mode_def, interpolation_def, kFragShader);
return create_fn(vert_src, frag_custom_zero_src);
};
#if GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
@@ -256,6 +328,7 @@ class GlTextureWarpAffineRunner
}
glUseProgram(program->id);
// uniforms
Eigen::Matrix<float, 4, 4, Eigen::RowMajor> eigen_mat(matrix.data());
if (IsMatrixVerticalFlipNeeded(gpu_origin_)) {
// @matrix describes affine transformation in terms of TOP LEFT origin, so
@@ -275,6 +348,10 @@ class GlTextureWarpAffineRunner
eigen_mat.transposeInPlace();
glUniformMatrix4fv(program->matrix_id, 1, GL_FALSE, eigen_mat.data());
if (interpolation_ == AffineTransformation::Interpolation::kCubic) {
glUniform2f(program->size_id, texture.width(), texture.height());
}
// vao
glBindVertexArray(vao_);
@@ -327,6 +404,7 @@ class GlTextureWarpAffineRunner
struct Program {
GLuint id;
GLint matrix_id;
GLint size_id;
};
std::shared_ptr<GlCalculatorHelper> gl_helper_;
GpuOrigin::Mode gpu_origin_;
@@ -335,6 +413,8 @@ class GlTextureWarpAffineRunner
Program program_;
std::optional<Program> program_custom_zero_;
GLuint framebuffer_ = 0;
AffineTransformation::Interpolation interpolation_ =
AffineTransformation::Interpolation::kLinear;
};
#undef GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
@@ -344,9 +424,10 @@ class GlTextureWarpAffineRunner
absl::StatusOr<std::unique_ptr<
AffineTransformation::Runner<GpuBuffer, std::unique_ptr<GpuBuffer>>>>
CreateAffineTransformationGlRunner(
std::shared_ptr<GlCalculatorHelper> gl_helper, GpuOrigin::Mode gpu_origin) {
auto runner =
absl::make_unique<GlTextureWarpAffineRunner>(gl_helper, gpu_origin);
std::shared_ptr<GlCalculatorHelper> gl_helper, GpuOrigin::Mode gpu_origin,
AffineTransformation::Interpolation interpolation) {
auto runner = absl::make_unique<GlTextureWarpAffineRunner>(
gl_helper, gpu_origin, interpolation);
MP_RETURN_IF_ERROR(runner->Init());
return runner;
}
@@ -29,7 +29,8 @@ absl::StatusOr<std::unique_ptr<AffineTransformation::Runner<
mediapipe::GpuBuffer, std::unique_ptr<mediapipe::GpuBuffer>>>>
CreateAffineTransformationGlRunner(
std::shared_ptr<mediapipe::GlCalculatorHelper> gl_helper,
mediapipe::GpuOrigin::Mode gpu_origin);
mediapipe::GpuOrigin::Mode gpu_origin,
AffineTransformation::Interpolation interpolation);
} // namespace mediapipe
@@ -39,9 +39,22 @@ cv::BorderTypes GetBorderModeForOpenCv(
}
}
int GetInterpolationForOpenCv(
AffineTransformation::Interpolation interpolation) {
switch (interpolation) {
case AffineTransformation::Interpolation::kLinear:
return cv::INTER_LINEAR;
case AffineTransformation::Interpolation::kCubic:
return cv::INTER_CUBIC;
}
}
class OpenCvRunner
: public AffineTransformation::Runner<ImageFrame, ImageFrame> {
public:
OpenCvRunner(AffineTransformation::Interpolation interpolation)
: interpolation_(GetInterpolationForOpenCv(interpolation)) {}
absl::StatusOr<ImageFrame> Run(
const ImageFrame& input, const std::array<float, 16>& matrix,
const AffineTransformation::Size& size,
@@ -142,19 +155,23 @@ class OpenCvRunner
cv::warpAffine(in_mat, out_mat, cv_affine_transform,
cv::Size(out_mat.cols, out_mat.rows),
/*flags=*/cv::INTER_LINEAR | cv::WARP_INVERSE_MAP,
/*flags=*/interpolation_ | cv::WARP_INVERSE_MAP,
GetBorderModeForOpenCv(border_mode));
return out_image;
}
private:
int interpolation_ = cv::INTER_LINEAR;
};
} // namespace
absl::StatusOr<
std::unique_ptr<AffineTransformation::Runner<ImageFrame, ImageFrame>>>
CreateAffineTransformationOpenCvRunner() {
return absl::make_unique<OpenCvRunner>();
CreateAffineTransformationOpenCvRunner(
AffineTransformation::Interpolation interpolation) {
return absl::make_unique<OpenCvRunner>(interpolation);
}
} // namespace mediapipe
@@ -25,7 +25,8 @@ namespace mediapipe {
absl::StatusOr<
std::unique_ptr<AffineTransformation::Runner<ImageFrame, ImageFrame>>>
CreateAffineTransformationOpenCvRunner();
CreateAffineTransformationOpenCvRunner(
AffineTransformation::Interpolation interpolation);
} // namespace mediapipe
@@ -81,7 +81,8 @@ class ImageCloneCalculator : public Node {
absl::Status Process(CalculatorContext* cc) override {
std::unique_ptr<Image> output;
const auto& input = *kIn(cc);
if (input.UsesGpu()) {
bool input_on_gpu = input.UsesGpu();
if (input_on_gpu) {
#if !MEDIAPIPE_DISABLE_GPU
// Create an output Image that co-owns the underlying texture buffer as
// the input Image.
@@ -97,15 +98,15 @@ class ImageCloneCalculator : public Node {
// Image. This ensures a correct life span of the shared pixel data.
output = std::make_unique<Image>(std::make_unique<mediapipe::ImageFrame>(
input.image_format(), input.width(), input.height(), input.step(),
const_cast<uint8*>(input.GetImageFrameSharedPtr()->PixelData()),
[packet_copy_ptr](uint8*) { delete packet_copy_ptr; }));
const_cast<uint8_t*>(input.GetImageFrameSharedPtr()->PixelData()),
[packet_copy_ptr](uint8_t*) { delete packet_copy_ptr; }));
}
if (output_on_gpu_) {
if (output_on_gpu_ && !input_on_gpu) {
#if !MEDIAPIPE_DISABLE_GPU
gpu_helper_.RunInGlContext([&output]() { output->ConvertToGpu(); });
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
} else if (!output_on_gpu_ && input_on_gpu) {
output->ConvertToCpu();
}
kOut(cc).Send(std::move(output));
@@ -22,6 +22,7 @@
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/vector.h"
@@ -53,24 +54,16 @@ enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
// range of [0, 1). Only the first channel of Alpha is used. Input & output Mat
// must be uchar.
template <typename AlphaType>
absl::Status MergeRGBA8Image(const cv::Mat input_mat, const cv::Mat& alpha_mat,
cv::Mat& output_mat) {
RET_CHECK_EQ(input_mat.rows, alpha_mat.rows);
RET_CHECK_EQ(input_mat.cols, alpha_mat.cols);
RET_CHECK_EQ(input_mat.rows, output_mat.rows);
RET_CHECK_EQ(input_mat.cols, output_mat.cols);
absl::Status CopyAlphaImage(const cv::Mat& alpha_mat, cv::Mat& output_mat) {
RET_CHECK_EQ(output_mat.rows, alpha_mat.rows);
RET_CHECK_EQ(output_mat.cols, alpha_mat.cols);
for (int i = 0; i < output_mat.rows; ++i) {
const uchar* in_ptr = input_mat.ptr<uchar>(i);
const AlphaType* alpha_ptr = alpha_mat.ptr<AlphaType>(i);
uchar* out_ptr = output_mat.ptr<uchar>(i);
for (int j = 0; j < output_mat.cols; ++j) {
const int out_idx = j * kNumChannelsRGBA;
const int in_idx = j * input_mat.channels();
const int alpha_idx = j * alpha_mat.channels();
out_ptr[out_idx + 0] = in_ptr[in_idx + 0];
out_ptr[out_idx + 1] = in_ptr[in_idx + 1];
out_ptr[out_idx + 2] = in_ptr[in_idx + 2];
if constexpr (std::is_same<AlphaType, uchar>::value) {
out_ptr[out_idx + 3] = alpha_ptr[alpha_idx + 0]; // channel 0 of mask
} else {
@@ -273,7 +266,7 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
// Setup source image
const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
const cv::Mat input_mat = mediapipe::formats::MatView(&input_frame);
const cv::Mat input_mat = formats::MatView(&input_frame);
if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) {
LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
}
@@ -281,38 +274,38 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
// Setup destination image
auto output_frame = absl::make_unique<ImageFrame>(
ImageFormat::SRGBA, input_mat.cols, input_mat.rows);
cv::Mat output_mat = mediapipe::formats::MatView(output_frame.get());
cv::Mat output_mat = formats::MatView(output_frame.get());
const bool has_alpha_mask = cc->Inputs().HasTag(kInputAlphaTag) &&
!cc->Inputs().Tag(kInputAlphaTag).IsEmpty();
const bool use_alpha_mask = alpha_value_ < 0 && has_alpha_mask;
// Setup alpha image and Update image in CPU.
// Copy rgb part of the image in CPU
if (input_mat.channels() == 3) {
cv::cvtColor(input_mat, output_mat, cv::COLOR_RGB2RGBA);
} else {
input_mat.copyTo(output_mat);
}
// Setup alpha image in CPU.
if (use_alpha_mask) {
const auto& alpha_mask = cc->Inputs().Tag(kInputAlphaTag).Get<ImageFrame>();
cv::Mat alpha_mat = mediapipe::formats::MatView(&alpha_mask);
cv::Mat alpha_mat = formats::MatView(&alpha_mask);
const bool alpha_is_float = CV_MAT_DEPTH(alpha_mat.type()) == CV_32F;
RET_CHECK(alpha_is_float || CV_MAT_DEPTH(alpha_mat.type()) == CV_8U);
if (alpha_is_float) {
MP_RETURN_IF_ERROR(
MergeRGBA8Image<float>(input_mat, alpha_mat, output_mat));
MP_RETURN_IF_ERROR(CopyAlphaImage<float>(alpha_mat, output_mat));
} else {
MP_RETURN_IF_ERROR(
MergeRGBA8Image<uchar>(input_mat, alpha_mat, output_mat));
MP_RETURN_IF_ERROR(CopyAlphaImage<uchar>(alpha_mat, output_mat));
}
} else {
const uchar alpha_value = std::min(std::max(0.0f, alpha_value_), 255.0f);
for (int i = 0; i < output_mat.rows; ++i) {
const uchar* in_ptr = input_mat.ptr<uchar>(i);
uchar* out_ptr = output_mat.ptr<uchar>(i);
for (int j = 0; j < output_mat.cols; ++j) {
const int out_idx = j * kNumChannelsRGBA;
const int in_idx = j * input_mat.channels();
out_ptr[out_idx + 0] = in_ptr[in_idx + 0];
out_ptr[out_idx + 1] = in_ptr[in_idx + 1];
out_ptr[out_idx + 2] = in_ptr[in_idx + 2];
out_ptr[out_idx + 3] = alpha_value; // use value from options
}
}
@@ -0,0 +1,156 @@
#include <cstdint>
#include "mediapipe/calculators/image/set_alpha_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "testing/base/public/benchmark.h"
namespace mediapipe {
namespace {
constexpr int input_width = 100;
constexpr int input_height = 100;
std::unique_ptr<ImageFrame> GetInputFrame(int width, int height, int channel) {
const int total_size = width * height * channel;
ImageFormat::Format image_format;
if (channel == 4) {
image_format = ImageFormat::SRGBA;
} else if (channel == 3) {
image_format = ImageFormat::SRGB;
} else {
image_format = ImageFormat::GRAY8;
}
auto input_frame = std::make_unique<ImageFrame>(image_format, width, height,
/*alignment_boundary =*/1);
for (int i = 0; i < total_size; ++i) {
input_frame->MutablePixelData()[i] = i % 256;
}
return input_frame;
}
// Test SetAlphaCalculator with RGB IMAGE input.
TEST(SetAlphaCalculatorTest, CpuRgb) {
auto calculator_node = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
R"pb(
calculator: "SetAlphaCalculator"
input_stream: "IMAGE:input_frames"
input_stream: "ALPHA:masks"
output_stream: "IMAGE:output_frames"
)pb");
CalculatorRunner runner(calculator_node);
// Input frames.
const auto input_frame = GetInputFrame(input_width, input_height, 3);
const auto mask_frame = GetInputFrame(input_width, input_height, 1);
auto input_frame_packet = MakePacket<ImageFrame>(std::move(*input_frame));
auto mask_frame_packet = MakePacket<ImageFrame>(std::move(*mask_frame));
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
input_frame_packet.At(Timestamp(1)));
runner.MutableInputs()->Tag("ALPHA").packets.push_back(
mask_frame_packet.At(Timestamp(1)));
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
EXPECT_EQ(outputs.NumEntries(), 1);
const auto& output_image = outputs.Tag("IMAGE").packets[0].Get<ImageFrame>();
// Generate ground truth (expected_mat).
const auto image = GetInputFrame(input_width, input_height, 3);
const auto input_mat = formats::MatView(image.get());
const auto mask = GetInputFrame(input_width, input_height, 1);
const auto mask_mat = formats::MatView(mask.get());
const std::array<cv::Mat, 2> input_mats = {input_mat, mask_mat};
cv::Mat expected_mat(input_width, input_height, CV_8UC4);
cv::mixChannels(input_mats, {expected_mat}, {0, 0, 1, 1, 2, 2, 3, 3});
cv::Mat output_mat = formats::MatView(&output_image);
double max_diff = cv::norm(expected_mat, output_mat, cv::NORM_INF);
EXPECT_FLOAT_EQ(max_diff, 0);
} // TEST
// Test SetAlphaCalculator with RGBA IMAGE input.
TEST(SetAlphaCalculatorTest, CpuRgba) {
auto calculator_node = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
R"pb(
calculator: "SetAlphaCalculator"
input_stream: "IMAGE:input_frames"
input_stream: "ALPHA:masks"
output_stream: "IMAGE:output_frames"
)pb");
CalculatorRunner runner(calculator_node);
// Input frames.
const auto input_frame = GetInputFrame(input_width, input_height, 4);
const auto mask_frame = GetInputFrame(input_width, input_height, 1);
auto input_frame_packet = MakePacket<ImageFrame>(std::move(*input_frame));
auto mask_frame_packet = MakePacket<ImageFrame>(std::move(*mask_frame));
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
input_frame_packet.At(Timestamp(1)));
runner.MutableInputs()->Tag("ALPHA").packets.push_back(
mask_frame_packet.At(Timestamp(1)));
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
EXPECT_EQ(outputs.NumEntries(), 1);
const auto& output_image = outputs.Tag("IMAGE").packets[0].Get<ImageFrame>();
// Generate ground truth (expected_mat).
const auto image = GetInputFrame(input_width, input_height, 4);
const auto input_mat = formats::MatView(image.get());
const auto mask = GetInputFrame(input_width, input_height, 1);
const auto mask_mat = formats::MatView(mask.get());
const std::array<cv::Mat, 2> input_mats = {input_mat, mask_mat};
cv::Mat expected_mat(input_width, input_height, CV_8UC4);
cv::mixChannels(input_mats, {expected_mat}, {0, 0, 1, 1, 2, 2, 4, 3});
cv::Mat output_mat = formats::MatView(&output_image);
double max_diff = cv::norm(expected_mat, output_mat, cv::NORM_INF);
EXPECT_FLOAT_EQ(max_diff, 0);
} // TEST
static void BM_SetAlpha3ChannelImage(benchmark::State& state) {
auto calculator_node = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
R"pb(
calculator: "SetAlphaCalculator"
input_stream: "IMAGE:input_frames"
input_stream: "ALPHA:masks"
output_stream: "IMAGE:output_frames"
)pb");
CalculatorRunner runner(calculator_node);
// Input frames.
const auto input_frame = GetInputFrame(input_width, input_height, 3);
const auto mask_frame = GetInputFrame(input_width, input_height, 1);
auto input_frame_packet = MakePacket<ImageFrame>(std::move(*input_frame));
auto mask_frame_packet = MakePacket<ImageFrame>(std::move(*mask_frame));
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
input_frame_packet.At(Timestamp(1)));
runner.MutableInputs()->Tag("ALPHA").packets.push_back(
mask_frame_packet.At(Timestamp(1)));
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
ASSERT_EQ(1, outputs.NumEntries());
for (const auto _ : state) {
MP_ASSERT_OK(runner.Run());
}
}
BENCHMARK(BM_SetAlpha3ChannelImage);
} // namespace
} // namespace mediapipe
@@ -53,6 +53,17 @@ AffineTransformation::BorderMode GetBorderMode(
}
}
AffineTransformation::Interpolation GetInterpolation(
mediapipe::WarpAffineCalculatorOptions::Interpolation interpolation) {
switch (interpolation) {
case mediapipe::WarpAffineCalculatorOptions::INTER_UNSPECIFIED:
case mediapipe::WarpAffineCalculatorOptions::INTER_LINEAR:
return AffineTransformation::Interpolation::kLinear;
case mediapipe::WarpAffineCalculatorOptions::INTER_CUBIC:
return AffineTransformation::Interpolation::kCubic;
}
}
template <typename ImageT>
class WarpAffineRunnerHolder {};
@@ -61,16 +72,22 @@ template <>
class WarpAffineRunnerHolder<ImageFrame> {
public:
using RunnerType = AffineTransformation::Runner<ImageFrame, ImageFrame>;
absl::Status Open(CalculatorContext* cc) { return absl::OkStatus(); }
absl::Status Open(CalculatorContext* cc) {
interpolation_ = GetInterpolation(
cc->Options<mediapipe::WarpAffineCalculatorOptions>().interpolation());
return absl::OkStatus();
}
absl::StatusOr<RunnerType*> GetRunner() {
if (!runner_) {
ASSIGN_OR_RETURN(runner_, CreateAffineTransformationOpenCvRunner());
ASSIGN_OR_RETURN(runner_,
CreateAffineTransformationOpenCvRunner(interpolation_));
}
return runner_.get();
}
private:
std::unique_ptr<RunnerType> runner_;
AffineTransformation::Interpolation interpolation_;
};
#endif // !MEDIAPIPE_DISABLE_OPENCV
@@ -85,12 +102,14 @@ class WarpAffineRunnerHolder<mediapipe::GpuBuffer> {
gpu_origin_ =
cc->Options<mediapipe::WarpAffineCalculatorOptions>().gpu_origin();
gl_helper_ = std::make_shared<mediapipe::GlCalculatorHelper>();
interpolation_ = GetInterpolation(
cc->Options<mediapipe::WarpAffineCalculatorOptions>().interpolation());
return gl_helper_->Open(cc);
}
absl::StatusOr<RunnerType*> GetRunner() {
if (!runner_) {
ASSIGN_OR_RETURN(
runner_, CreateAffineTransformationGlRunner(gl_helper_, gpu_origin_));
ASSIGN_OR_RETURN(runner_, CreateAffineTransformationGlRunner(
gl_helper_, gpu_origin_, interpolation_));
}
return runner_.get();
}
@@ -99,6 +118,7 @@ class WarpAffineRunnerHolder<mediapipe::GpuBuffer> {
mediapipe::GpuOrigin::Mode gpu_origin_;
std::shared_ptr<mediapipe::GlCalculatorHelper> gl_helper_;
std::unique_ptr<RunnerType> runner_;
AffineTransformation::Interpolation interpolation_;
};
#endif // !MEDIAPIPE_DISABLE_GPU
@@ -31,6 +31,13 @@ message WarpAffineCalculatorOptions {
BORDER_REPLICATE = 2;
}
// Pixel sampling interpolation methods. See @interpolation.
enum Interpolation {
INTER_UNSPECIFIED = 0;
INTER_LINEAR = 1;
INTER_CUBIC = 2;
}
// Pixel extrapolation method.
// When converting image to tensor it may happen that tensor needs to read
// pixels outside image boundaries. Border mode helps to specify how such
@@ -43,4 +50,10 @@ message WarpAffineCalculatorOptions {
// to be flipped vertically as tensors are expected to start at top.
// (DEFAULT or unset interpreted as CONVENTIONAL.)
optional GpuOrigin.Mode gpu_origin = 2;
// Sampling method for neighboring pixels.
// INTER_LINEAR (bilinear) linearly interpolates from the nearest 4 neighbors.
// INTER_CUBIC (bicubic) interpolates a small neighborhood with cubic weights.
// INTER_UNSPECIFIED or unset interpreted as INTER_LINEAR.
optional Interpolation interpolation = 3;
}
@@ -63,7 +63,8 @@ void RunTest(const std::string& graph_text, const std::string& tag,
const cv::Mat& input, cv::Mat expected_result,
float similarity_threshold, std::array<float, 16> matrix,
int out_width, int out_height,
absl::optional<AffineTransformation::BorderMode> border_mode) {
std::optional<AffineTransformation::BorderMode> border_mode,
std::optional<AffineTransformation::Interpolation> interpolation) {
std::string border_mode_str;
if (border_mode) {
switch (*border_mode) {
@@ -75,8 +76,20 @@ void RunTest(const std::string& graph_text, const std::string& tag,
break;
}
}
std::string interpolation_str;
if (interpolation) {
switch (*interpolation) {
case AffineTransformation::Interpolation::kLinear:
interpolation_str = "interpolation: INTER_LINEAR";
break;
case AffineTransformation::Interpolation::kCubic:
interpolation_str = "interpolation: INTER_CUBIC";
break;
}
}
auto graph_config = mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(graph_text, /*$0=*/border_mode_str));
absl::Substitute(graph_text, /*$0=*/border_mode_str,
/*$1=*/interpolation_str));
std::vector<Packet> output_packets;
tool::AddVectorSink("output_image", &graph_config, &output_packets);
@@ -132,7 +145,8 @@ struct SimilarityConfig {
void RunTest(cv::Mat input, cv::Mat expected_result,
const SimilarityConfig& similarity, std::array<float, 16> matrix,
int out_width, int out_height,
absl::optional<AffineTransformation::BorderMode> border_mode) {
std::optional<AffineTransformation::BorderMode> border_mode,
std::optional<AffineTransformation::Interpolation> interpolation) {
RunTest(R"(
input_stream: "input_image"
input_stream: "output_size"
@@ -146,12 +160,13 @@ void RunTest(cv::Mat input, cv::Mat expected_result,
options {
[mediapipe.WarpAffineCalculatorOptions.ext] {
$0 # border mode
$1 # interpolation
}
}
}
)",
"cpu", input, expected_result, similarity.threshold_on_cpu, matrix,
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
RunTest(R"(
input_stream: "input_image"
@@ -171,6 +186,7 @@ void RunTest(cv::Mat input, cv::Mat expected_result,
options {
[mediapipe.WarpAffineCalculatorOptions.ext] {
$0 # border mode
$1 # interpolation
}
}
}
@@ -181,7 +197,7 @@ void RunTest(cv::Mat input, cv::Mat expected_result,
}
)",
"cpu_image", input, expected_result, similarity.threshold_on_cpu,
matrix, out_width, out_height, border_mode);
matrix, out_width, out_height, border_mode, interpolation);
RunTest(R"(
input_stream: "input_image"
@@ -201,6 +217,7 @@ void RunTest(cv::Mat input, cv::Mat expected_result,
options {
[mediapipe.WarpAffineCalculatorOptions.ext] {
$0 # border mode
$1 # interpolation
gpu_origin: TOP_LEFT
}
}
@@ -212,7 +229,7 @@ void RunTest(cv::Mat input, cv::Mat expected_result,
}
)",
"gpu", input, expected_result, similarity.threshold_on_gpu, matrix,
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
RunTest(R"(
input_stream: "input_image"
@@ -237,6 +254,7 @@ void RunTest(cv::Mat input, cv::Mat expected_result,
options {
[mediapipe.WarpAffineCalculatorOptions.ext] {
$0 # border mode
$1 # interpolation
gpu_origin: TOP_LEFT
}
}
@@ -253,7 +271,7 @@ void RunTest(cv::Mat input, cv::Mat expected_result,
}
)",
"gpu_image", input, expected_result, similarity.threshold_on_gpu,
matrix, out_width, out_height, border_mode);
matrix, out_width, out_height, border_mode, interpolation);
}
std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
@@ -287,10 +305,11 @@ TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspect) {
int out_height = 256;
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode = {};
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.82},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectBorderZero) {
@@ -312,10 +331,11 @@ TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectBorderZero) {
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kZero;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.81},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectWithRotation) {
@@ -337,10 +357,11 @@ TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectWithRotation) {
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kReplicate;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.77},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectWithRotationBorderZero) {
@@ -362,10 +383,11 @@ TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectWithRotationBorderZero) {
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kZero;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.75},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, MediumSubRectWithRotation) {
@@ -386,10 +408,11 @@ TEST(WarpAffineCalculatorTest, MediumSubRectWithRotation) {
bool keep_aspect_ratio = false;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kReplicate;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.81},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, MediumSubRectWithRotationBorderZero) {
@@ -411,10 +434,38 @@ TEST(WarpAffineCalculatorTest, MediumSubRectWithRotationBorderZero) {
bool keep_aspect_ratio = false;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kZero;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.80},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, MediumSubRectWithRotationBorderZeroInterpCubic) {
mediapipe::NormalizedRect roi;
roi.set_x_center(0.65f);
roi.set_y_center(0.4f);
roi.set_width(0.5f);
roi.set_height(0.5f);
roi.set_rotation(M_PI * -45.0f / 180.0f);
auto input = GetRgb(
"/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/input.jpg");
auto expected_output = GetRgb(
"/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"medium_sub_rect_with_rotation_border_zero_interp_cubic.png");
int out_width = 256;
int out_height = 256;
bool keep_aspect_ratio = false;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kZero;
std::optional<AffineTransformation::Interpolation> interpolation =
AffineTransformation::Interpolation::kCubic;
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.78},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, LargeSubRect) {
@@ -435,10 +486,11 @@ TEST(WarpAffineCalculatorTest, LargeSubRect) {
bool keep_aspect_ratio = false;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kReplicate;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.95},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, LargeSubRectBorderZero) {
@@ -459,10 +511,11 @@ TEST(WarpAffineCalculatorTest, LargeSubRectBorderZero) {
bool keep_aspect_ratio = false;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kZero;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.92},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspect) {
@@ -483,10 +536,11 @@ TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspect) {
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kReplicate;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.97},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectBorderZero) {
@@ -508,10 +562,11 @@ TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectBorderZero) {
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kZero;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.97},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectWithRotation) {
@@ -532,10 +587,11 @@ TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectWithRotation) {
int out_height = 128;
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode = {};
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.91},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectWithRotationBorderZero) {
@@ -557,10 +613,11 @@ TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectWithRotationBorderZero) {
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kZero;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.88},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, NoOp) {
@@ -581,10 +638,11 @@ TEST(WarpAffineCalculatorTest, NoOp) {
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kReplicate;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.99},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
TEST(WarpAffineCalculatorTest, NoOpBorderZero) {
@@ -605,10 +663,11 @@ TEST(WarpAffineCalculatorTest, NoOpBorderZero) {
bool keep_aspect_ratio = true;
std::optional<AffineTransformation::BorderMode> border_mode =
AffineTransformation::BorderMode::kZero;
std::optional<AffineTransformation::Interpolation> interpolation = {};
RunTest(input, expected_output,
{.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.99},
GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
out_width, out_height, border_mode);
out_width, out_height, border_mode, interpolation);
}
} // namespace
+6 -11
View File
@@ -12,25 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
proto_library(
mediapipe_proto_library(
name = "callback_packet_calculator_proto",
srcs = ["callback_packet_calculator.proto"],
visibility = ["//mediapipe/framework:__subpackages__"],
deps = ["//mediapipe/framework:calculator_proto"],
)
mediapipe_cc_proto_library(
name = "callback_packet_calculator_cc_proto",
srcs = ["callback_packet_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//mediapipe/framework:__subpackages__"],
deps = [":callback_packet_calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
+90 -55
View File
@@ -237,7 +237,9 @@ cc_library(
cc_test(
name = "bert_preprocessor_calculator_test",
srcs = ["bert_preprocessor_calculator_test.cc"],
data = ["//mediapipe/tasks/testdata/text:bert_text_classifier_models"],
data = [
"//mediapipe/tasks/testdata/text:bert_text_classifier_models",
],
linkopts = ["-ldl"],
deps = [
":bert_preprocessor_calculator",
@@ -250,7 +252,7 @@ cc_test(
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_sentencepiece//src:sentencepiece_processor",
"@com_google_sentencepiece//src:sentencepiece_processor", # fixdeps: keep
],
)
@@ -300,7 +302,7 @@ cc_test(
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_sentencepiece//src:sentencepiece_processor",
"@com_google_sentencepiece//src:sentencepiece_processor", # fixdeps: keep
],
)
@@ -442,12 +444,12 @@ cc_library(
tags = ["nomac"],
deps = [
":inference_calculator_interface",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"//mediapipe/framework/deps:file_path",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/util/tflite:tflite_gpu_runner",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@org_tensorflow//tensorflow/lite:framework_stable",
] + select({
"//conditions:default": [],
@@ -465,10 +467,6 @@ cc_library(
"-x objective-c++",
"-fobjc-arc", # enable reference-counting
],
linkopts = [
"-framework CoreVideo",
"-framework MetalKit",
],
tags = ["ios"],
deps = [
"inference_calculator_interface",
@@ -484,7 +482,13 @@ cc_library(
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
"@org_tensorflow//tensorflow/lite/delegates/gpu/metal:buffer_convert",
],
] + select({
"//mediapipe:apple": [
"//third_party/apple_frameworks:CoreVideo",
"//third_party/apple_frameworks:MetalKit",
],
"//conditions:default": [],
}),
alwayslink = 1,
)
@@ -603,8 +607,8 @@ cc_library(
cc_library(
name = "inference_calculator",
deps = [
":inference_calculator_interface",
":inference_calculator_cpu",
":inference_calculator_interface",
] + select({
"//conditions:default": [":inference_calculator_gl_if_compute_shader_available"],
":platform_ios_with_gpu": [":inference_calculator_metal"],
@@ -642,11 +646,11 @@ cc_library(
deps = [
":tensor_converter_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework:port",
"//mediapipe/util:resource_util",
] + select({
"//mediapipe/gpu:disable_gpu": [],
@@ -664,16 +668,16 @@ cc_library(
"//mediapipe/gpu:gpu_buffer",
],
"//mediapipe:ios": [
"//mediapipe/gpu:MPPMetalUtil",
"//mediapipe/gpu:MPPMetalHelper",
"//mediapipe/gpu:MPPMetalUtil",
"//mediapipe/objc:mediapipe_framework_ios",
],
"//mediapipe:macos": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:shader_util",
],
}),
)
@@ -719,29 +723,28 @@ cc_library(
"//conditions:default": [],
}),
features = ["-layering_check"], # allow depending on tensors_to_detections_calculator_gpu_deps
linkopts = select({
"//mediapipe:apple": [
"-framework CoreVideo",
"-framework MetalKit",
],
"//conditions:default": [],
}),
deps = [
":tensors_to_detections_calculator_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:location",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span",
] + selects.with_or({
":compute_shader_unavailable": [],
"//conditions:default": [":tensors_to_detections_calculator_gpu_deps"],
}) + select({
"//mediapipe:apple": [
"//third_party/apple_frameworks:CoreVideo",
"//third_party/apple_frameworks:MetalKit",
],
"//conditions:default": [],
}),
alwayslink = 1,
)
@@ -751,8 +754,8 @@ cc_library(
visibility = ["//visibility:private"],
deps = select({
"//mediapipe:ios": [
"//mediapipe/gpu:MPPMetalUtil",
"//mediapipe/gpu:MPPMetalHelper",
"//mediapipe/gpu:MPPMetalUtil",
],
"//mediapipe:macos": [],
"//conditions:default": [
@@ -898,17 +901,17 @@ cc_library(
}),
deps = [
":tensors_to_classification_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:location",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/util:label_map_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/container:node_hash_map",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:location",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/formats:tensor",
"//mediapipe/util:label_map_cc_proto",
"//mediapipe/util:resource_util",
] + select({
"//mediapipe:android": [
"//mediapipe/util/android/file/base",
@@ -968,6 +971,8 @@ cc_library(
":image_to_tensor_converter",
":image_to_tensor_utils",
":loose_headers",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
@@ -976,8 +981,6 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
"//mediapipe/gpu:gpu_origin_cc_proto",
] + select({
"//mediapipe/gpu:disable_gpu": [],
@@ -985,6 +988,11 @@ cc_library(
}) + select({
"//mediapipe/framework/port:disable_opencv": [],
"//conditions:default": [":image_to_tensor_converter_opencv"],
}) + select({
"//mediapipe/framework/port:enable_halide": [
":image_to_tensor_converter_frame_buffer",
],
"//conditions:default": [],
}),
alwayslink = 1,
)
@@ -997,17 +1005,20 @@ cc_library(
":image_to_tensor_converter_gl_buffer",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_service",
],
"//mediapipe:apple": [
":image_to_tensor_converter_metal",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:MPPMetalHelper",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_service",
],
"//conditions:default": [
":image_to_tensor_converter_gl_buffer",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_service",
],
}),
)
@@ -1065,7 +1076,10 @@ cc_test(
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
] + select({
"//mediapipe:apple": [],
"//conditions:default": ["//mediapipe/gpu:gl_context"],
}),
)
cc_library(
@@ -1112,6 +1126,26 @@ cc_library(
],
)
cc_library(
name = "image_to_tensor_converter_frame_buffer",
srcs = ["image_to_tensor_converter_frame_buffer.cc"],
hdrs = ["image_to_tensor_converter_frame_buffer.h"],
deps = [
":image_to_tensor_converter",
":image_to_tensor_utils",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework/formats:frame_buffer",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:frame_buffer_view",
"//mediapipe/util/frame_buffer:frame_buffer_util",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "image_to_tensor_converter_gl_buffer",
srcs = ["image_to_tensor_converter_gl_buffer.cc"],
@@ -1157,16 +1191,16 @@ cc_library(
":image_to_tensor_converter",
":image_to_tensor_converter_gl_utils",
":image_to_tensor_utils",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/framework/formats:image",
"//mediapipe/gpu:shader_util",
"@com_google_absl//absl/strings",
],
}),
)
@@ -1179,10 +1213,10 @@ cc_library(
deps = ["//mediapipe/framework:port"] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_context",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_context",
],
}),
)
@@ -1210,15 +1244,15 @@ cc_library(
"//mediapipe:apple": [
":image_to_tensor_converter",
":image_to_tensor_utils",
"//mediapipe/gpu:MPPMetalHelper",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/framework/formats:image",
"//mediapipe/gpu:MPPMetalHelper",
"//mediapipe/gpu:gpu_buffer_format",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:types",
],
@@ -1239,8 +1273,6 @@ cc_library(
}),
deps = [
":image_to_tensor_calculator_cc_proto",
"@com_google_absl//absl/status",
"@com_google_absl//absl/types:optional",
"//mediapipe/framework/api2:packet",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:image",
@@ -1249,6 +1281,8 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_origin_cc_proto",
"@com_google_absl//absl/status",
"@com_google_absl//absl/types:optional",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": ["//mediapipe/gpu:gpu_buffer"],
@@ -1298,20 +1332,20 @@ cc_library(
}),
deps = [
":tensors_to_segmentation_calculator_cc_proto",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/types:span",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span",
"@org_tensorflow//tensorflow/lite:framework",
"//mediapipe/framework/port:statusor",
] + selects.with_or({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
@@ -1325,6 +1359,7 @@ cc_library(
"//mediapipe:ios": [
"//mediapipe/gpu:MPPMetalUtil",
"//mediapipe/gpu:MPPMetalHelper",
"//third_party/apple_frameworks:MetalKit",
],
"//conditions:default": [
"@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
@@ -192,7 +192,7 @@ class AudioToTensorCalculator : public Node {
DftTensorFormat dft_tensor_format_;
Timestamp initial_timestamp_ = Timestamp::Unstarted();
int64 cumulative_input_samples_ = 0;
int64_t cumulative_input_samples_ = 0;
Timestamp next_output_timestamp_ = Timestamp::Unstarted();
double source_sample_rate_ = -1;
@@ -163,20 +163,21 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
}
void CheckTimestampsOutputPackets(
std::vector<int64> expected_timestamp_values) {
std::vector<int64_t> expected_timestamp_values) {
ASSERT_EQ(num_iterations_, timestamps_packets_.size());
for (int i = 0; i < timestamps_packets_.size(); ++i) {
const auto& p = timestamps_packets_[i];
MP_ASSERT_OK(p.ValidateAsType<std::vector<Timestamp>>());
auto output_timestamps = p.Get<std::vector<Timestamp>>();
int64 base_timestamp = i * Timestamp::kTimestampUnitsPerSecond;
int64_t base_timestamp = i * Timestamp::kTimestampUnitsPerSecond;
std::vector<Timestamp> expected_timestamps;
expected_timestamps.resize(expected_timestamp_values.size());
std::transform(
expected_timestamp_values.begin(), expected_timestamp_values.end(),
expected_timestamps.begin(), [base_timestamp](int64 v) -> Timestamp {
return Timestamp(v + base_timestamp);
});
std::transform(expected_timestamp_values.begin(),
expected_timestamp_values.end(),
expected_timestamps.begin(),
[base_timestamp](int64_t v) -> Timestamp {
return Timestamp(v + base_timestamp);
});
EXPECT_EQ(expected_timestamps, output_timestamps);
EXPECT_EQ(p.Timestamp(), expected_timestamps.back());
}
@@ -379,7 +380,7 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
}
void CheckTensorsOutputPackets(int sample_offset, int num_packets,
int64 timestamp_interval,
int64_t timestamp_interval,
bool output_last_at_close) {
ASSERT_EQ(num_packets, tensors_packets_.size());
for (int i = 0; i < num_packets; ++i) {
@@ -550,7 +551,7 @@ class AudioToTensorCalculatorFftTest : public ::testing::Test {
protected:
// Creates an audio matrix containing a single sample of 1.0 at a specified
// offset.
std::unique_ptr<Matrix> CreateImpulseSignalData(int64 num_samples,
std::unique_ptr<Matrix> CreateImpulseSignalData(int64_t num_samples,
int impulse_offset_idx) {
Matrix impulse = Matrix::Zero(1, num_samples);
impulse(0, impulse_offset_idx) = 1.0;
@@ -121,31 +121,39 @@ class BertPreprocessorCalculator : public Node {
private:
std::unique_ptr<tasks::text::tokenizers::Tokenizer> tokenizer_;
// The max sequence length accepted by the BERT model.
// The max sequence length accepted by the BERT model if its input tensors
// are static.
int bert_max_seq_len_ = 2;
// Indices of the three input tensors for the BERT model. They should form the
// set {0, 1, 2}.
int input_ids_tensor_index_ = 0;
int segment_ids_tensor_index_ = 1;
int input_masks_tensor_index_ = 2;
// Whether the model's input tensor shapes are dynamic.
bool has_dynamic_input_tensors_ = false;
// Applies `tokenizer_` to the `input_text` to generate a vector of tokens.
// This util prepends "[CLS]" and appends "[SEP]" to the input tokens and
// clips the vector of tokens to have length at most `bert_max_seq_len_`.
// clips the vector of tokens to have length at most `bert_max_seq_len_` if
// the input tensors are static.
std::vector<std::string> TokenizeInputText(absl::string_view input_text);
// Processes the `input_tokens` to generate the three input tensors for the
// BERT model.
// Processes the `input_tokens` to generate the three input tensors of size
// `tensor_size` for the BERT model.
std::vector<Tensor> GenerateInputTensors(
const std::vector<std::string>& input_tokens);
const std::vector<std::string>& input_tokens, int tensor_size);
};
absl::Status BertPreprocessorCalculator::UpdateContract(
CalculatorContract* cc) {
const auto& options =
cc->Options<mediapipe::BertPreprocessorCalculatorOptions>();
RET_CHECK(options.has_bert_max_seq_len()) << "bert_max_seq_len is required";
RET_CHECK_GE(options.bert_max_seq_len(), 2)
<< "bert_max_seq_len must be at least 2";
if (options.has_dynamic_input_tensors()) {
return absl::OkStatus();
} else {
RET_CHECK(options.has_bert_max_seq_len()) << "bert_max_seq_len is required";
RET_CHECK_GE(options.bert_max_seq_len(), 2)
<< "bert_max_seq_len must be at least 2";
}
return absl::OkStatus();
}
@@ -178,12 +186,17 @@ absl::Status BertPreprocessorCalculator::Open(CalculatorContext* cc) {
const auto& options =
cc->Options<mediapipe::BertPreprocessorCalculatorOptions>();
bert_max_seq_len_ = options.bert_max_seq_len();
has_dynamic_input_tensors_ = options.has_dynamic_input_tensors();
return absl::OkStatus();
}
absl::Status BertPreprocessorCalculator::Process(CalculatorContext* cc) {
kTensorsOut(cc).Send(
GenerateInputTensors(TokenizeInputText(kTextIn(cc).Get())));
int tensor_size = bert_max_seq_len_;
std::vector<std::string> input_tokens = TokenizeInputText(kTextIn(cc).Get());
if (has_dynamic_input_tensors_) {
tensor_size = input_tokens.size();
}
kTensorsOut(cc).Send(GenerateInputTensors(input_tokens, tensor_size));
return absl::OkStatus();
}
@@ -197,8 +210,11 @@ std::vector<std::string> BertPreprocessorCalculator::TokenizeInputText(
// Offset by 2 to account for [CLS] and [SEP]
int input_tokens_size =
std::min(bert_max_seq_len_,
static_cast<int>(tokenizer_result.subwords.size()) + 2);
static_cast<int>(tokenizer_result.subwords.size()) + 2;
// For static shapes, truncate the input tokens to `bert_max_seq_len_`.
if (!has_dynamic_input_tensors_) {
input_tokens_size = std::min(bert_max_seq_len_, input_tokens_size);
}
std::vector<std::string> input_tokens;
input_tokens.reserve(input_tokens_size);
input_tokens.push_back(std::string(kClassifierToken));
@@ -210,16 +226,16 @@ std::vector<std::string> BertPreprocessorCalculator::TokenizeInputText(
}
std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
const std::vector<std::string>& input_tokens) {
std::vector<int32_t> input_ids(bert_max_seq_len_, 0);
std::vector<int32_t> segment_ids(bert_max_seq_len_, 0);
std::vector<int32_t> input_masks(bert_max_seq_len_, 0);
const std::vector<std::string>& input_tokens, int tensor_size) {
std::vector<int32_t> input_ids(tensor_size, 0);
std::vector<int32_t> segment_ids(tensor_size, 0);
std::vector<int32_t> input_masks(tensor_size, 0);
// Convert tokens back into ids and set mask
for (int i = 0; i < input_tokens.size(); ++i) {
tokenizer_->LookupId(input_tokens[i], &input_ids[i]);
input_masks[i] = 1;
}
// |<--------bert_max_seq_len_--------->|
// |<-----------tensor_size------------>|
// input_ids [CLS] s1 s2... sn [SEP] 0 0... 0
// segment_ids 0 0 0... 0 0 0 0... 0
// input_masks 1 1 1... 1 1 0 0... 0
@@ -228,7 +244,7 @@ std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
input_tensors.reserve(kNumInputTensorsForBert);
for (int i = 0; i < kNumInputTensorsForBert; ++i) {
input_tensors.push_back(
{Tensor::ElementType::kInt32, Tensor::Shape({bert_max_seq_len_})});
{Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})});
}
std::memcpy(input_tensors[input_ids_tensor_index_]
.GetCpuWriteView()
@@ -24,6 +24,10 @@ message BertPreprocessorCalculatorOptions {
optional BertPreprocessorCalculatorOptions ext = 462509271;
}
// The maximum input sequence length for the calculator's BERT model.
// The maximum input sequence length for the calculator's BERT model. Used
// if the model's input tensors have static shape.
optional int32 bert_max_seq_len = 1;
// Whether the BERT model's input tensors have dynamic shape.
optional bool has_dynamic_input_tensors = 2;
}
@@ -43,7 +43,8 @@ constexpr absl::string_view kTestModelPath =
"mediapipe/tasks/testdata/text/bert_text_classifier.tflite";
absl::StatusOr<std::vector<std::vector<int>>> RunBertPreprocessorCalculator(
absl::string_view text, absl::string_view model_path) {
absl::string_view text, absl::string_view model_path,
bool has_dynamic_input_tensors = false, int tensor_size = kBertMaxSeqLen) {
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(R"(
input_stream: "text"
@@ -56,11 +57,12 @@ absl::StatusOr<std::vector<std::vector<int>>> RunBertPreprocessorCalculator(
options {
[mediapipe.BertPreprocessorCalculatorOptions.ext] {
bert_max_seq_len: $0
has_dynamic_input_tensors: $1
}
}
}
)",
kBertMaxSeqLen));
tensor_size, has_dynamic_input_tensors));
std::vector<Packet> output_packets;
tool::AddVectorSink("tensors", &graph_config, &output_packets);
@@ -92,13 +94,13 @@ absl::StatusOr<std::vector<std::vector<int>>> RunBertPreprocessorCalculator(
}
std::vector<std::vector<int>> results;
for (int i = 0; i < kNumInputTensorsForBert; i++) {
for (int i = 0; i < tensor_vec.size(); i++) {
const Tensor& tensor = tensor_vec[i];
if (tensor.element_type() != Tensor::ElementType::kInt32) {
return absl::InvalidArgumentError("Expected tensor element type kInt32");
}
auto* buffer = tensor.GetCpuReadView().buffer<int>();
std::vector<int> buffer_view(buffer, buffer + kBertMaxSeqLen);
std::vector<int> buffer_view(buffer, buffer + tensor_size);
results.push_back(buffer_view);
}
MP_RETURN_IF_ERROR(graph.CloseAllPacketSources());
@@ -34,6 +34,8 @@
#if !MEDIAPIPE_DISABLE_OPENCV
#include "mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h"
#elif MEDIAPIPE_ENABLE_HALIDE
#include "mediapipe/calculators/tensor/image_to_tensor_converter_frame_buffer.h"
#endif
#if !MEDIAPIPE_DISABLE_GPU
@@ -45,9 +47,11 @@
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_service.h"
#else
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_service.h"
#endif // MEDIAPIPE_METAL_ENABLED
#endif // !MEDIAPIPE_DISABLE_GPU
@@ -147,7 +151,7 @@ class ImageToTensorCalculator : public Node {
#if MEDIAPIPE_METAL_ENABLED
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
#else
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
cc->UseService(kGpuService).Optional();
#endif // MEDIAPIPE_METAL_ENABLED
#endif // MEDIAPIPE_DISABLE_GPU
@@ -271,10 +275,19 @@ class ImageToTensorCalculator : public Node {
CreateOpenCvConverter(
cc, GetBorderMode(options_.border_mode()),
GetOutputTensorType(/*uses_gpu=*/false, params_)));
// TODO: FrameBuffer-based converter needs to call GetGpuBuffer()
// to get access to a FrameBuffer view. Investigate if GetGpuBuffer() can be
// made available even with MEDIAPIPE_DISABLE_GPU set.
#elif MEDIAPIPE_ENABLE_HALIDE
ASSIGN_OR_RETURN(cpu_converter_,
CreateFrameBufferConverter(
cc, GetBorderMode(options_.border_mode()),
GetOutputTensorType(/*uses_gpu=*/false, params_)));
#else
LOG(FATAL) << "Cannot create image to tensor opencv converter since "
"MEDIAPIPE_DISABLE_OPENCV is defined.";
#endif // !MEDIAPIPE_DISABLE_OPENCV
LOG(FATAL) << "Cannot create image to tensor CPU converter since "
"MEDIAPIPE_DISABLE_OPENCV is defined and "
"MEDIAPIPE_ENABLE_HALIDE is not defined.";
#endif // !MEDIAPIPE_DISABLE_HALIDE
}
}
return absl::OkStatus();
@@ -41,6 +41,10 @@
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/util/image_test_utils.h"
#if !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED
#include "mediapipe/gpu/gl_context.h"
#endif // !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED
namespace mediapipe {
namespace {
@@ -159,12 +163,12 @@ void RunTestWithInputImagePacket(
EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kInt8);
tensor_mat = cv::Mat(expected_result.rows, expected_result.cols,
channels == 1 ? CV_8SC1 : CV_8SC3,
const_cast<int8*>(view.buffer<int8>()));
const_cast<int8_t*>(view.buffer<int8_t>()));
} else {
EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kUInt8);
tensor_mat = cv::Mat(expected_result.rows, expected_result.cols,
channels == 1 ? CV_8UC1 : CV_8UC3,
const_cast<uint8*>(view.buffer<uint8>()));
const_cast<uint8_t*>(view.buffer<uint8_t>()));
}
} else {
EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kFloat32);
@@ -206,14 +210,14 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
Packet MakeImageFramePacket(cv::Mat input) {
ImageFrame input_image(GetImageFormat(input.channels()), input.cols,
input.rows, input.step, input.data, [](uint8*) {});
input.rows, input.step, input.data, [](uint8_t*) {});
return MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0));
}
Packet MakeImagePacket(cv::Mat input) {
mediapipe::Image input_image(std::make_shared<mediapipe::ImageFrame>(
GetImageFormat(input.channels()), input.cols, input.rows, input.step,
input.data, [](uint8*) {}));
input.data, [](uint8_t*) {}));
return MakePacket<mediapipe::Image>(std::move(input_image)).At(Timestamp(0));
}
@@ -507,5 +511,79 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRangeAndUseInputImageDims) {
/*tensor_width=*/std::nullopt, /*tensor_height=*/std::nullopt,
/*keep_aspect=*/false, BorderMode::kZero, roi);
}
TEST(ImageToTensorCalculatorTest, CanBeUsedWithoutGpuServiceSet) {
auto graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
input_stream: "input_image"
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor"
options {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_float_range { min: 0.0f max: 1.0f }
}
}
}
)pb");
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.DisallowServiceDefaultInitialization());
MP_ASSERT_OK(graph.StartRun({}));
auto image_frame =
std::make_shared<ImageFrame>(ImageFormat::SRGBA, 128, 256, 4);
Image image = Image(std::move(image_frame));
Packet packet = MakePacket<Image>(std::move(image));
MP_ASSERT_OK(
graph.AddPacketToInputStream("input_image", packet.At(Timestamp(1))));
MP_ASSERT_OK(graph.WaitUntilIdle());
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_ASSERT_OK(graph.WaitUntilDone());
}
#if !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED
TEST(ImageToTensorCalculatorTest,
FailsGracefullyWhenGpuServiceNeededButNotAvailable) {
auto graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
input_stream: "input_image"
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor"
options {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_float_range { min: 0.0f max: 1.0f }
}
}
}
)pb");
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.DisallowServiceDefaultInitialization());
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK_AND_ASSIGN(auto context,
GlContext::Create(nullptr, /*create_thread=*/true));
Packet packet;
context->Run([&packet]() {
auto image_frame =
std::make_shared<ImageFrame>(ImageFormat::SRGBA, 128, 256, 4);
Image image = Image(std::move(image_frame));
// Ensure image is available on GPU to force ImageToTensorCalculator to
// run on GPU.
ASSERT_TRUE(image.ConvertToGpu());
packet = MakePacket<Image>(std::move(image));
});
MP_ASSERT_OK(
graph.AddPacketToInputStream("input_image", packet.At(Timestamp(1))));
EXPECT_THAT(graph.WaitUntilIdle(),
StatusIs(absl::StatusCode::kInternal,
HasSubstr("GPU service not available")));
}
#endif // !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED
} // namespace
} // namespace mediapipe

Some files were not shown because too many files have changed in this diff Show More