diff --git a/README.md b/README.md index e7a9c10d..f5a0bcee 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,13 @@ ## ML Solutions in MediaPipe * [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) +* [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md) * [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) * [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) * [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md) ![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif) +![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif) ![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif) ![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif) ![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif) @@ -23,7 +25,7 @@ Follow these [instructions](mediapipe/docs/install.md). ## Getting started -See mobile and desktop [examples](mediapipe/docs/examples.md). +See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md). ## Documentation [MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev) @@ -41,6 +43,7 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172) ## Events +* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/) * [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/) * [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038) * [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/) diff --git a/WORKSPACE b/WORKSPACE index ca87c83c..0ed4579b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -149,11 +149,10 @@ new_local_repository( http_archive( name = "android_opencv", - sha256 = "056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b", build_file = "@//third_party:opencv_android.BUILD", strip_prefix = "OpenCV-android-sdk", type = "zip", - url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip", + url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip", ) # After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will @@ -184,13 +183,18 @@ maven_install( artifacts = [ "androidx.annotation:annotation:aar:1.1.0", "androidx.appcompat:appcompat:aar:1.1.0-rc01", + "androidx.camera:camera-core:aar:1.0.0-alpha06", + "androidx.camera:camera-camera2:aar:1.0.0-alpha06", "androidx.constraintlayout:constraintlayout:aar:1.1.3", "androidx.core:core:aar:1.1.0-rc03", "androidx.legacy:legacy-support-v4:aar:1.0.0", "androidx.recyclerview:recyclerview:aar:1.1.0-beta02", "com.google.android.material:material:aar:1.0.0-rc01", ], - repositories = ["https://dl.google.com/dl/android/maven2"], + repositories = [ + "https://dl.google.com/dl/android/maven2", + "https://repo1.maven.org/maven2", + ], ) maven_server( diff --git a/mediapipe/calculators/image/scale_image_calculator.proto b/mediapipe/calculators/image/scale_image_calculator.proto index 2fc782a4..0f0d5aae 100644 --- a/mediapipe/calculators/image/scale_image_calculator.proto +++ b/mediapipe/calculators/image/scale_image_calculator.proto @@ -36,7 +36,8 @@ message ScaleImageCalculatorOptions { // If ratio is positive, crop the image to this minimum and maximum // aspect ratio (preserving the center of the frame). This is done - // before scaling. + // before scaling. The string must contain "/", so to disable cropping, + // set both to "0/1". // For example, for a min_aspect_ratio of "9/16" and max of "16/9" the // following cropping will occur: // 1920x1080 (which is 16:9) is not cropped diff --git a/mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc b/mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc index 5de7b0c0..eed7ca9d 100644 --- a/mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc +++ b/mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc @@ -85,7 +85,7 @@ class TFRecordReaderCalculator : public CalculatorBase { tensorflow::io::RecordReader reader(file.get(), tensorflow::io::RecordReaderOptions()); tensorflow::uint64 offset = 0; - std::string example_str; + tensorflow::tstring example_str; const int target_idx = cc->InputSidePackets().HasTag(kRecordIndex) ? cc->InputSidePackets().Tag(kRecordIndex).Get() @@ -98,7 +98,7 @@ class TFRecordReaderCalculator : public CalculatorBase { if (current_idx == target_idx) { if (cc->OutputSidePackets().HasTag(kExampleTag)) { tensorflow::Example tf_example; - tf_example.ParseFromString(example_str); + tf_example.ParseFromArray(example_str.data(), example_str.size()); cc->OutputSidePackets() .Tag(kExampleTag) .Set(MakePacket(std::move(tf_example))); diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.cc b/mediapipe/calculators/tflite/tflite_inference_calculator.cc index 0abb4832..9ca1c281 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_inference_calculator.cc @@ -64,6 +64,28 @@ typedef id GpuTensor; size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT } // namespace +#if defined(MEDIAPIPE_EDGE_TPU) +#include "edgetpu.h" + +// Creates and returns an Edge TPU interpreter to run the given edgetpu model. +std::unique_ptr BuildEdgeTpuInterpreter( + const tflite::FlatBufferModel& model, + tflite::ops::builtin::BuiltinOpResolver* resolver, + edgetpu::EdgeTpuContext* edgetpu_context) { + resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp()); + std::unique_ptr interpreter; + if (tflite::InterpreterBuilder(model, *resolver)(&interpreter) != kTfLiteOk) { + std::cerr << "Failed to build edge TPU interpreter." << std::endl; + } + interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context); + interpreter->SetNumThreads(1); + if (interpreter->AllocateTensors() != kTfLiteOk) { + std::cerr << "Failed to allocate edge TPU tensors." << std::endl; + } + return interpreter; +} +#endif // MEDIAPIPE_EDGE_TPU + // TfLiteInferenceCalculator File Layout: // * Header // * Core @@ -162,6 +184,11 @@ class TfLiteInferenceCalculator : public CalculatorBase { TFLBufferConvert* converter_from_BPHWC4_ = nil; #endif +#if defined(MEDIAPIPE_EDGE_TPU) + std::shared_ptr edgetpu_context_ = + edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice(); +#endif + std::string model_path_ = ""; bool gpu_inference_ = false; bool gpu_input_ = false; @@ -425,6 +452,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); #endif delegate_ = nullptr; } +#if defined(MEDIAPIPE_EDGE_TPU) + edgetpu_context_.reset(); +#endif return ::mediapipe::OkStatus(); } @@ -458,16 +488,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); model_ = tflite::FlatBufferModel::BuildFromFile(model_path_.c_str()); RET_CHECK(model_); + tflite::ops::builtin::BuiltinOpResolver op_resolver; if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) { - const auto& op_resolver = - cc->InputSidePackets() - .Tag("CUSTOM_OP_RESOLVER") - .Get(); - tflite::InterpreterBuilder(*model_, op_resolver)(&interpreter_); - } else { - const tflite::ops::builtin::BuiltinOpResolver op_resolver; - tflite::InterpreterBuilder(*model_, op_resolver)(&interpreter_); + op_resolver = cc->InputSidePackets() + .Tag("CUSTOM_OP_RESOLVER") + .Get(); } +#if defined(MEDIAPIPE_EDGE_TPU) + interpreter_ = + BuildEdgeTpuInterpreter(*model_, &op_resolver, edgetpu_context_.get()); +#else + tflite::InterpreterBuilder(*model_, op_resolver)(&interpreter_); +#endif // MEDIAPIPE_EDGE_TPU RET_CHECK(interpreter_); diff --git a/mediapipe/calculators/util/labels_to_render_data_calculator.cc b/mediapipe/calculators/util/labels_to_render_data_calculator.cc index a7f51729..722f4480 100644 --- a/mediapipe/calculators/util/labels_to_render_data_calculator.cc +++ b/mediapipe/calculators/util/labels_to_render_data_calculator.cc @@ -93,6 +93,7 @@ REGISTER_CALCULATOR(LabelsToRenderDataCalculator); } ::mediapipe::Status LabelsToRenderDataCalculator::Open(CalculatorContext* cc) { + cc->SetOffset(TimestampDiff(0)); options_ = cc->Options(); num_colors_ = options_.color_size(); label_height_px_ = std::ceil(options_.font_height_px() * kFontHeightScale); diff --git a/mediapipe/docs/android_archive_library.md b/mediapipe/docs/android_archive_library.md index 5351e58f..0fed0e40 100644 --- a/mediapipe/docs/android_archive_library.md +++ b/mediapipe/docs/android_archive_library.md @@ -92,7 +92,7 @@ project. MediaPipe depends on OpenCV, you will need to copy the precompiled OpenCV so files into app/src/main/jniLibs. You can download the official OpenCV Android SDK from - [here](https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip) + [here](https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip) and run: ```bash diff --git a/mediapipe/docs/examples.md b/mediapipe/docs/examples.md index 4e279e93..15cfa6ce 100644 --- a/mediapipe/docs/examples.md +++ b/mediapipe/docs/examples.md @@ -157,3 +157,20 @@ how to use MediaPipe with a TFLite model for hair segmentation on desktop using GPU with live video from a webcam. * [Desktop GPU](./hair_segmentation_desktop.md) + +## Google Coral (machine learning acceleration with Google EdgeTPU) + +Below are code samples on how to run MediaPipe on Google Coral Dev Board. + +### Object Detection on Coral + +[Object Detection on Coral with Webcam](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md) +shows how to run quantized object detection TFlite model accelerated with +EdgeTPU on +[Google Coral Dev Board](https://coral.withgoogle.com/products/dev-board). + +### Face Detection on Coral + +[Face Detection on Coral with Webcam](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md) +shows how to use quantized face detection TFlite model accelerated with EdgeTPU +on [Google Coral Dev Board](https://coral.withgoogle.com/products/dev-board). diff --git a/mediapipe/docs/hello_world_android.md b/mediapipe/docs/hello_world_android.md index 35f62834..eb186d65 100644 --- a/mediapipe/docs/hello_world_android.md +++ b/mediapipe/docs/hello_world_android.md @@ -629,7 +629,7 @@ to load both dependencies: static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } ``` diff --git a/mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif b/mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif new file mode 100644 index 00000000..e1c068f2 Binary files /dev/null and b/mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif differ diff --git a/mediapipe/docs/multi_hand_tracking_desktop.md b/mediapipe/docs/multi_hand_tracking_desktop.md index d80707f9..295c1016 100644 --- a/mediapipe/docs/multi_hand_tracking_desktop.md +++ b/mediapipe/docs/multi_hand_tracking_desktop.md @@ -156,7 +156,7 @@ node { output_stream: "multi_hand_rects" node_options: { [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { - min_similarity_threshold: 0.1 + min_similarity_threshold: 0.5 } } } diff --git a/mediapipe/docs/multi_hand_tracking_mobile_gpu.md b/mediapipe/docs/multi_hand_tracking_mobile_gpu.md index b57a6631..53a09d96 100644 --- a/mediapipe/docs/multi_hand_tracking_mobile_gpu.md +++ b/mediapipe/docs/multi_hand_tracking_mobile_gpu.md @@ -219,7 +219,7 @@ node { output_stream: "multi_hand_rects" node_options: { [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { - min_similarity_threshold: 0.1 + min_similarity_threshold: 0.5 } } } diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/MainActivity.java index 9c45cbd3..15708d6a 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/MainActivity.java @@ -47,7 +47,7 @@ public class MainActivity extends AppCompatActivity { static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } // {@link SurfaceTexture} where the camera-preview frames can be accessed. diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/MainActivity.java index 05ef0e75..fab13221 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/MainActivity.java @@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity { static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } // {@link SurfaceTexture} where the camera-preview frames can be accessed. diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/MainActivity.java index 91e53345..5990c6c4 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/MainActivity.java @@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity { static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } // {@link SurfaceTexture} where the camera-preview frames can be accessed. diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/MainActivity.java index 16022d9d..11e1fe1a 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/MainActivity.java @@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity { static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } // {@link SurfaceTexture} where the camera-preview frames can be accessed. diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/MainActivity.java index cc95bbbf..2d7c84a2 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/MainActivity.java @@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity { static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } // {@link SurfaceTexture} where the camera-preview frames can be accessed. diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java index 5e69f62e..a907ea4e 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java @@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity { static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } // {@link SurfaceTexture} where the camera-preview frames can be accessed. diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java index cef13854..7e541ea4 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java @@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity { static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } // {@link SurfaceTexture} where the camera-preview frames can be accessed. diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/MainActivity.java index c6bdb8d5..de7f478a 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/MainActivity.java @@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity { static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } // {@link SurfaceTexture} where the camera-preview frames can be accessed. diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java index c436ef7f..39d51d58 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java @@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity { static { // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java4"); + System.loadLibrary("opencv_java3"); } // {@link SurfaceTexture} where the camera-preview frames can be accessed. diff --git a/mediapipe/examples/coral/BUILD b/mediapipe/examples/coral/BUILD new file mode 100644 index 00000000..1515140d --- /dev/null +++ b/mediapipe/examples/coral/BUILD @@ -0,0 +1,56 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = [ + "//visibility:public", +]) + +# Graph Runner + +cc_library( + name = "demo_run_graph_main", + srcs = ["demo_run_graph_main.cc"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:commandlineflags", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_video", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + ], +) + +# Demos + +cc_binary( + name = "object_detection_cpu", + deps = [ + "//mediapipe/examples/coral:demo_run_graph_main", + "//mediapipe/graphs/object_detection:desktop_tflite_calculators", + ], +) + +cc_binary( + name = "face_detection_cpu", + deps = [ + "//mediapipe/examples/coral:demo_run_graph_main", + "//mediapipe/graphs/face_detection:desktop_tflite_calculators", + ], +) diff --git a/mediapipe/examples/coral/Dockerfile b/mediapipe/examples/coral/Dockerfile new file mode 100644 index 00000000..1640f57d --- /dev/null +++ b/mediapipe/examples/coral/Dockerfile @@ -0,0 +1,87 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#==== ! Prerequisite ! ==== +# $ sh mediapipe/examples/coral/setup.sh +#==== + +# for opencv 3.2 default +FROM ubuntu:18.04 + +MAINTAINER + +WORKDIR /mediapipe + +ENV DEBIAN_FRONTEND=noninteractive + +# Install MediaPipe & Coral deps + +COPY update_sources.sh / +RUN /update_sources.sh + +RUN dpkg --add-architecture armhf +RUN dpkg --add-architecture arm64 +RUN apt-get update && apt-get install -y \ + build-essential \ + crossbuild-essential-arm64 \ + libusb-1.0-0-dev \ + libusb-1.0-0-dev:arm64 \ + zlib1g-dev \ + zlib1g-dev:arm64 \ + pkg-config \ + zip \ + unzip \ + curl \ + wget \ + git \ + python \ + python-pip \ + python3-pip \ + vim-common \ + ca-certificates \ + emacs \ + software-properties-common && \ + add-apt-repository -y ppa:openjdk-r/ppa && \ + apt-get update && apt-get install -y openjdk-8-jdk + +RUN pip install --upgrade setuptools +RUN pip install future +RUN pip3 install six + +COPY . /mediapipe/ + +# Install bazel + +ARG BAZEL_VERSION=0.29.1 +RUN mkdir /bazel && \ + wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \ + wget --no-check-certificate -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + chmod +x /bazel/installer.sh && \ + /bazel/installer.sh && \ + rm -f /bazel/installer.sh + +# OpenCV (3.2 default in 18.04) + +RUN apt-get update && apt-get install -y libopencv-dev + +# Opencv libs copied from coral device into opencv32_arm64_libs + +RUN cp opencv32_arm64_libs/* /usr/lib/aarch64-linux-gnu/. + +# Edge tpu header and lib + +RUN git clone https://github.com/google-coral/edgetpu.git /edgetpu +RUN cp /edgetpu/libedgetpu/direct/aarch64/libedgetpu.so.1.0 /usr/lib/aarch64-linux-gnu/libedgetpu.so + +# See mediapipe/examples/coral/README.md to finish setup diff --git a/mediapipe/examples/coral/README.md b/mediapipe/examples/coral/README.md new file mode 100644 index 00000000..6712a1e0 --- /dev/null +++ b/mediapipe/examples/coral/README.md @@ -0,0 +1,137 @@ +# Coral Dev Board Setup (experimental) + +**Dislaimer**: Running MediaPipe on Coral is experimental, and this process may not be exact and is subject to change. These instructions have only been tested on the coral dev board with OS version _mendel day_, and may vary for different devices and workstations. + +This file describes how to prepare a Google Coral Dev Board and setup a linux Docker container for building MediaPipe applications that run on Edge TPU. + +## Before creating the Docker + +* (on host machine) run _setup.sh_ from MediaPipe root directory + + sh mediapipe/examples/coral/setup.sh + +* Setup the coral device via [here](https://coral.withgoogle.com/docs/dev-board/get-started/), and ensure the _mdt_ command works + +* (on coral device) prepare MediaPipe + + cd ~ + sudo apt-get install git + git clone https://github.com/google/mediapipe.git + mkdir mediapipe/bazel-bin + +* (on coral device) install opencv 3.2 + + sudo apt-get update && apt-get install -y libopencv-dev + +* (on coral device) find all opencv libs + + find /usr/lib/aarch64-linux-gnu/ -name 'libopencv*so' + +* (on host machine) copy core opencv libs from coral device to a local folder inside MediaPipe checkout: + + # in root level mediapipe folder # + mdt pull /usr/lib/aarch64-linux-gnu/libopencv_core.so opencv32_arm64_libs + mdt pull /usr/lib/aarch64-linux-gnu/libopencv_calib3d.so opencv32_arm64_libs + mdt pull /usr/lib/aarch64-linux-gnu/libopencv_features2d.so opencv32_arm64_libs + mdt pull /usr/lib/aarch64-linux-gnu/libopencv_highgui.so opencv32_arm64_libs + mdt pull /usr/lib/aarch64-linux-gnu/libopencv_imgcodecs.so opencv32_arm64_libs + mdt pull /usr/lib/aarch64-linux-gnu/libopencv_imgproc.so opencv32_arm64_libs + mdt pull /usr/lib/aarch64-linux-gnu/libopencv_video.so opencv32_arm64_libs + mdt pull /usr/lib/aarch64-linux-gnu/libopencv_videoio.so opencv32_arm64_libs + +* (on host machine) Create and start the docker environment + + # from mediapipe root level directory # + docker build -t coral . + docker run -it --name coral coral:latest + +## Inside the Docker environment + +* Update library paths in /mediapipe/third_party/opencv_linux.BUILD + + (replace 'x86_64-linux-gnu' with 'aarch64-linux-gnu') + + "lib/aarch64-linux-gnu/libopencv_core.so", + "lib/aarch64-linux-gnu/libopencv_calib3d.so", + "lib/aarch64-linux-gnu/libopencv_features2d.so", + "lib/aarch64-linux-gnu/libopencv_highgui.so", + "lib/aarch64-linux-gnu/libopencv_imgcodecs.so", + "lib/aarch64-linux-gnu/libopencv_imgproc.so", + "lib/aarch64-linux-gnu/libopencv_video.so", + "lib/aarch64-linux-gnu/libopencv_videoio.so", + +* Attempt to build hello world (to download external deps) + + bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world + +* Edit /mediapipe/bazel-mediapipe/external/com_github_glog_glog/src/signalhandler.cc + + on line 78, replace + + return (void*)context->PC_FROM_UCONTEXT; + + with + + return NULL; + +* Edit /edgetpu/libedgetpu/BUILD + + to add this build target + + cc_library( + name = "lib", + srcs = [ + "libedgetpu.so", + ], + visibility = ["//visibility:public"], + ) + +* Edit *tflite_inference_calculator.cc* BUILD rules: + + sed -i 's/\":tflite_inference_calculator_cc_proto\",/\":tflite_inference_calculator_cc_proto\",\n\t\"@edgetpu\/\/:header\",\n\t\"@libedgetpu\/\/:lib\",/g' mediapipe/calculators/tflite/BUILD + + The above command should add + + "@edgetpu//:header", + "@libedgetpu//:lib", + + to the _deps_ of tflite_inference_calculator.cc + +#### Now try cross-compiling for device + +* Object detection demo + + bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:object_detection_cpu + + Copy object_detection_cpu binary to the MediaPipe checkout on the coral device + + # outside docker env, open new terminal on host machine # + docker ps + docker cp :/mediapipe/bazel-bin/mediapipe/examples/coral/object_detection_cpu /tmp/. + mdt push /tmp/object_detection_cpu /home/mendel/mediapipe/bazel-bin/. + +* Face detection demo + + bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:face_detection_cpu + + Copy face_detection_cpu binary to the MediaPipe checkout on the coral device + + # outside docker env, open new terminal on host machine # + docker ps + docker cp :/mediapipe/bazel-bin/mediapipe/examples/coral/face_detection_cpu /tmp/. + mdt push /tmp/face_detection_cpu /home/mendel/mediapipe/bazel-bin/. + +## On the coral device (with display) + + # Object detection + cd ~/mediapipe + chmod +x bazel-bin/object_detection_cpu + export GLOG_logtostderr=1 + bazel-bin/object_detection_cpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt + + # Face detection + cd ~/mediapipe + chmod +x bazel-bin/face_detection_cpu + export GLOG_logtostderr=1 + bazel-bin/face_detection_cpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt + diff --git a/mediapipe/examples/coral/WORKSPACE b/mediapipe/examples/coral/WORKSPACE new file mode 100644 index 00000000..28112f95 --- /dev/null +++ b/mediapipe/examples/coral/WORKSPACE @@ -0,0 +1,313 @@ +workspace(name = "mediapipe") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +skylib_version = "0.8.0" +http_archive( + name = "bazel_skylib", + type = "tar.gz", + url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version), + sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e", +) +load("@bazel_skylib//lib:versions.bzl", "versions") +versions.check(minimum_bazel_version = "0.24.1") + +# ABSL cpp library. +http_archive( + name = "com_google_absl", + # Head commit on 2019-04-12. + # TODO: Switch to the latest absl version when the problem gets + # fixed. + urls = [ + "https://github.com/abseil/abseil-cpp/archive/a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a.tar.gz", + ], + sha256 = "d437920d1434c766d22e85773b899c77c672b8b4865d5dc2cd61a29fdff3cf03", + strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a", +) + +http_archive( + name = "rules_cc", + strip_prefix = "rules_cc-master", + urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"], +) + +# GoogleTest/GoogleMock framework. Used by most unit-tests. +http_archive( + name = "com_google_googletest", + urls = ["https://github.com/google/googletest/archive/master.zip"], + strip_prefix = "googletest-master", +) + +# Google Benchmark library. +http_archive( + name = "com_google_benchmark", + urls = ["https://github.com/google/benchmark/archive/master.zip"], + strip_prefix = "benchmark-master", + build_file = "@//third_party:benchmark.BUILD", +) + +# gflags needed by glog +http_archive( + name = "com_github_gflags_gflags", + sha256 = "6e16c8bc91b1310a44f3965e616383dbda48f83e8c1eaa2370a215057b00cabe", + strip_prefix = "gflags-77592648e3f3be87d6c7123eb81cbad75f9aef5a", + urls = [ + "https://mirror.bazel.build/github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz", + "https://github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz", + ], +) + +# glog +http_archive( + name = "com_github_glog_glog", + url = "https://github.com/google/glog/archive/v0.3.5.zip", + sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8", + strip_prefix = "glog-0.3.5", + build_file = "@//third_party:glog.BUILD", + patches = [ + "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff" + ], + patch_args = [ + "-p1", + ], +) + +# libyuv +http_archive( + name = "libyuv", + urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"], + build_file = "@//third_party:libyuv.BUILD", +) + +http_archive( + name = "com_google_protobuf_javalite", + sha256 = "79d102c61e2a479a0b7e5fc167bcfaa4832a0c6aad4a75fa7da0480564931bcc", + strip_prefix = "protobuf-384989534b2246d413dbcd750744faab2607b516", + urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"], +) + +http_archive( + name = "com_google_audio_tools", + strip_prefix = "multichannel-audio-tools-master", + urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"], +) + +# Needed by TensorFlow +http_archive( + name = "io_bazel_rules_closure", + sha256 = "e0a111000aeed2051f29fcc7a3f83be3ad8c6c93c186e64beb1ad313f0c7f9f9", + strip_prefix = "rules_closure-cf1e44edb908e9616030cc83d085989b8e6cd6df", + urls = [ + "http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz", + "https://github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz", # 2019-04-04 + ], +) + +# 2019-11-12 +_TENSORFLOW_GIT_COMMIT = "a5f9bcd64453ff3d1f64cb4da4786db3d2da7f82" +_TENSORFLOW_SHA256= "f2b6f2ab2ffe63e86eccd3ce4bea6b7197383d726638dfeeebcdc1e7de73f075" +http_archive( + name = "org_tensorflow", + urls = [ + "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, + "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, + ], + strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT, + sha256 = _TENSORFLOW_SHA256, +) + +load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace") +tf_workspace(tf_repo_name = "org_tensorflow") + +# Please run +# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \ +# libopencv-imgproc-dev libopencv-video-dev +new_local_repository( + name = "linux_opencv", + build_file = "@//third_party:opencv_linux.BUILD", + path = "/usr", +) + +new_local_repository( + name = "linux_ffmpeg", + build_file = "@//third_party:ffmpeg_linux.BUILD", + path = "/usr" +) + +# Please run $ brew install opencv@3 +new_local_repository( + name = "macos_opencv", + build_file = "@//third_party:opencv_macos.BUILD", + path = "/usr", +) + +new_local_repository( + name = "macos_ffmpeg", + build_file = "@//third_party:ffmpeg_macos.BUILD", + path = "/usr", +) + +http_archive( + name = "android_opencv", + sha256 = "056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b", + build_file = "@//third_party:opencv_android.BUILD", + strip_prefix = "OpenCV-android-sdk", + type = "zip", + url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip", +) + +# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will +# trigger duplicate symbol errors in the linking stage of building a mediapipe ios app. +# To get a higher version of OpenCV for iOS, opencv2.framework needs to be built from source with +# '-DBUILD_PROTOBUF=OFF -DBUILD_opencv_dnn=OFF'. +http_archive( + name = "ios_opencv", + sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2", + build_file = "@//third_party:opencv_ios.BUILD", + type = "zip", + url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip", +) + +RULES_JVM_EXTERNAL_TAG = "2.2" +RULES_JVM_EXTERNAL_SHA = "f1203ce04e232ab6fdd81897cf0ff76f2c04c0741424d192f28e65ae752ce2d6" + +http_archive( + name = "rules_jvm_external", + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, + sha256 = RULES_JVM_EXTERNAL_SHA, + url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, +) + +load("@rules_jvm_external//:defs.bzl", "maven_install") + +maven_install( + artifacts = [ + "androidx.annotation:annotation:aar:1.1.0", + "androidx.appcompat:appcompat:aar:1.1.0-rc01", + "androidx.constraintlayout:constraintlayout:aar:1.1.3", + "androidx.core:core:aar:1.1.0-rc03", + "androidx.legacy:legacy-support-v4:aar:1.0.0", + "androidx.recyclerview:recyclerview:aar:1.1.0-beta02", + "com.google.android.material:material:aar:1.0.0-rc01", + ], + repositories = ["https://dl.google.com/dl/android/maven2"], +) + +maven_server( + name = "google_server", + url = "https://dl.google.com/dl/android/maven2", +) + +maven_jar( + name = "androidx_lifecycle", + artifact = "androidx.lifecycle:lifecycle-common:2.0.0", + sha1 = "e070ffae07452331bc5684734fce6831d531785c", + server = "google_server", +) + +maven_jar( + name = "androidx_concurrent_futures", + artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03", + sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae", + server = "google_server", +) + +maven_jar( + name = "com_google_guava_android", + artifact = "com.google.guava:guava:27.0.1-android", + sha1 = "b7e1c37f66ef193796ccd7ea6e80c2b05426182d", +) + +maven_jar( + name = "com_google_common_flogger", + artifact = "com.google.flogger:flogger:0.3.1", + sha1 = "585030fe1ec709760cbef997a459729fb965df0e", +) + +maven_jar( + name = "com_google_common_flogger_system_backend", + artifact = "com.google.flogger:flogger-system-backend:0.3.1", + sha1 = "287b569d76abcd82f9de87fe41829fbc7ebd8ac9", +) + +maven_jar( + name = "com_google_code_findbugs", + artifact = "com.google.code.findbugs:jsr305:3.0.2", + sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d", +) + +# You may run setup_android.sh to install Android SDK and NDK. +android_ndk_repository( + name = "androidndk", +) + +android_sdk_repository( + name = "androidsdk", +) + +# iOS basic build deps. + +http_archive( + name = "build_bazel_rules_apple", + sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e", + strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3", + url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", +) + +load( + "@build_bazel_rules_apple//apple:repositories.bzl", + "apple_rules_dependencies", +) + +apple_rules_dependencies() + +load( + "@build_bazel_rules_swift//swift:repositories.bzl", + "swift_rules_dependencies", +) + +swift_rules_dependencies() + +load( + "@build_bazel_apple_support//lib:repositories.bzl", + "apple_support_dependencies", +) + +apple_support_dependencies() + +# More iOS deps. + +http_archive( + name = "google_toolbox_for_mac", + url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip", + sha256 = "e3ac053813c989a88703556df4dc4466e424e30d32108433ed6beaec76ba4fdc", + strip_prefix = "google-toolbox-for-mac-2.2.1", + build_file = "@//third_party:google_toolbox_for_mac.BUILD", +) + + +# Coral +#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}') +#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}') +http_archive( + name = "coral_crosstool", + sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb", + strip_prefix = "crosstool-9e00d5be43bf001f883b5700f5d04882fea00229", + urls = [ + "https://github.com/google-coral/crosstool/archive/9e00d5be43bf001f883b5700f5d04882fea00229.tar.gz", + ], +) +load("@coral_crosstool//:configure.bzl", "cc_crosstool") +cc_crosstool(name = "crosstool") + +# EdgeTPU +new_local_repository( + name = "edgetpu", + path = "/edgetpu/libedgetpu", + build_file = "/edgetpu/libedgetpu/BUILD" +) +new_local_repository( + name = "libedgetpu", + path = "/usr/lib/aarch64-linux-gnu", + build_file = "/edgetpu/libedgetpu/BUILD" +) diff --git a/mediapipe/examples/coral/demo_run_graph_main.cc b/mediapipe/examples/coral/demo_run_graph_main.cc new file mode 100644 index 00000000..fb5e8692 --- /dev/null +++ b/mediapipe/examples/coral/demo_run_graph_main.cc @@ -0,0 +1,151 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// An example of sending OpenCV webcam frames into a MediaPipe graph. + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/commandlineflags.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" + +constexpr char kInputStream[] = "input_video"; +constexpr char kOutputStream[] = "output_video"; +constexpr char kWindowName[] = "MediaPipe"; + +DEFINE_string( + calculator_graph_config_file, "", + "Name of file containing text format CalculatorGraphConfig proto."); +DEFINE_string(input_video_path, "", + "Full path of video to load. " + "If not provided, attempt to use a webcam."); +DEFINE_string(output_video_path, "", + "Full path of where to save result (.mp4 only). " + "If not provided, show result in a window."); + +::mediapipe::Status RunMPPGraph() { + std::string calculator_graph_config_contents; + MP_RETURN_IF_ERROR(mediapipe::file::GetContents( + FLAGS_calculator_graph_config_file, &calculator_graph_config_contents)); + LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; + mediapipe::CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie( + calculator_graph_config_contents); + + LOG(INFO) << "Initialize the calculator graph."; + mediapipe::CalculatorGraph graph; + MP_RETURN_IF_ERROR(graph.Initialize(config)); + + LOG(INFO) << "Initialize the camera or load the video."; + cv::VideoCapture capture; + const bool load_video = !FLAGS_input_video_path.empty(); + if (load_video) { + capture.open(FLAGS_input_video_path); + } else { + capture.open(0); + } + RET_CHECK(capture.isOpened()); + + cv::VideoWriter writer; + const bool save_video = !FLAGS_output_video_path.empty(); + if (save_video) { + LOG(INFO) << "Prepare video writer."; + cv::Mat test_frame; + capture.read(test_frame); // Consume first frame. + capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning. + writer.open(FLAGS_output_video_path, + mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 + capture.get(cv::CAP_PROP_FPS), test_frame.size()); + RET_CHECK(writer.isOpened()); + } else { + cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); + capture.set(cv::CAP_PROP_FRAME_WIDTH, 640); + capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + capture.set(cv::CAP_PROP_AUTOFOCUS, 0); + capture.set(cv::CAP_PROP_FOCUS, 1); + capture.set(cv::CAP_PROP_FPS, 30); + } + + LOG(INFO) << "Start running the calculator graph."; + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, + graph.AddOutputStreamPoller(kOutputStream)); + MP_RETURN_IF_ERROR(graph.StartRun({})); + + LOG(INFO) << "Start grabbing and processing frames."; + size_t frame_timestamp = 0; + bool grab_frames = true; + while (grab_frames) { + // Capture opencv camera or video frame. + cv::Mat camera_frame_raw; + capture >> camera_frame_raw; + if (camera_frame_raw.empty()) break; // End of video. + cv::Mat camera_frame; + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + if (!load_video) { + cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); + } + + // Wrap Mat into an ImageFrame. + auto input_frame = absl::make_unique( + mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, + mediapipe::ImageFrame::kDefaultAlignmentBoundary); + cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); + camera_frame.copyTo(input_frame_mat); + + // Send image packet into the graph. + MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( + kInputStream, mediapipe::Adopt(input_frame.release()) + .At(mediapipe::Timestamp(frame_timestamp++)))); + + // Get the graph result packet, or stop if that fails. + mediapipe::Packet packet; + if (!poller.Next(&packet)) break; + auto& output_frame = packet.Get(); + + // Convert back to opencv for display or saving. + cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); + cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); + if (save_video) { + writer.write(output_frame_mat); + } else { + cv::imshow(kWindowName, output_frame_mat); + // Press any key to exit. + const int pressed_key = cv::waitKey(5); + if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; + } + } + + LOG(INFO) << "Shutting down."; + if (writer.isOpened()) writer.release(); + MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); + return graph.WaitUntilDone(); +} + +int main(int argc, char** argv) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + ::mediapipe::Status run_status = RunMPPGraph(); + if (!run_status.ok()) { + LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + } else { + LOG(INFO) << "Success!"; + } + return 0; +} diff --git a/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt b/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt new file mode 100644 index 00000000..eaae2f90 --- /dev/null +++ b/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt @@ -0,0 +1,189 @@ +# MediaPipe graph that performs face detection with TensorFlow Lite on CPU. +# Used in the examples in +# mediapipe/examples/coral:face_detection_cpu. + +# Images on GPU coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for +# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish +# generating the corresponding detections before it passes through another +# image. All images that come in while waiting are dropped, limiting the number +# of in-flight images between this calculator and +# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between +# from queuing up incoming images and data excessively, which leads to increased +# latency and memory usage, unwanted in real-time mobile applications. It also +# eliminates unnecessarily computation, e.g., a transformed image produced by +# ImageTransformationCalculator may get dropped downstream if the subsequent +# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy +# processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:detections" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Transforms the input image on CPU to a 128x128 image. To scale the input +# image, the scale_mode option is set to FIT to preserve the aspect ratio, +# resulting in potential letterboxing in the transformed image. +node: { + calculator: "ImageTransformationCalculator" + input_stream: "IMAGE:throttled_input_video" + output_stream: "IMAGE:transformed_input_video_cpu" + output_stream: "LETTERBOX_PADDING:letterbox_padding" + options: { + [mediapipe.ImageTransformationCalculatorOptions.ext] { + output_width: 128 + output_height: 128 + scale_mode: FIT + } + } +} + +# Converts the transformed input image on CPU into an image tensor stored as a +# TfLiteTensor. +node { + calculator: "TfLiteConverterCalculator" + input_stream: "IMAGE:transformed_input_video_cpu" + output_stream: "TENSORS:image_tensor" + options: { + [mediapipe.TfLiteConverterCalculatorOptions.ext] { + use_quantized_tensors: true + } + } +} + +# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "TfLiteInferenceCalculator" + input_stream: "TENSORS:image_tensor" + output_stream: "TENSORS:detection_tensors" + options: { + [mediapipe.TfLiteInferenceCalculatorOptions.ext] { + model_path: "mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite" + } + } +} + +# Generates a single side packet containing a vector of SSD anchors based on +# the specification in the options. +node { + calculator: "SsdAnchorsCalculator" + output_side_packet: "anchors" + options: { + [mediapipe.SsdAnchorsCalculatorOptions.ext] { + num_layers: 4 + min_scale: 0.1484375 + max_scale: 0.75 + input_size_height: 128 + input_size_width: 128 + anchor_offset_x: 0.5 + anchor_offset_y: 0.5 + strides: 8 + strides: 16 + strides: 16 + strides: 16 + aspect_ratios: 1.0 + fixed_anchor_size: true + } + } +} + +# Decodes the detection tensors generated by the TensorFlow Lite model, based on +# the SSD anchors and the specification in the options, into a vector of +# detections. Each detection describes a detected object. +node { + calculator: "TfLiteTensorsToDetectionsCalculator" + input_stream: "TENSORS:detection_tensors" + input_side_packet: "ANCHORS:anchors" + output_stream: "DETECTIONS:detections" + options: { + [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] { + num_classes: 1 + num_boxes: 896 + num_coords: 16 + box_coord_offset: 0 + keypoint_coord_offset: 4 + num_keypoints: 6 + num_values_per_keypoint: 2 + sigmoid_score: true + score_clipping_thresh: 100.0 + reverse_output_order: true + x_scale: 128.0 + y_scale: 128.0 + h_scale: 128.0 + w_scale: 128.0 + min_score_thresh: 0.75 + } + } +} + +# Performs non-max suppression to remove excessive detections. +node { + calculator: "NonMaxSuppressionCalculator" + input_stream: "detections" + output_stream: "filtered_detections" + options: { + [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { + min_suppression_threshold: 0.3 + overlap_type: INTERSECTION_OVER_UNION + algorithm: WEIGHTED + return_empty_detections: true + } + } +} + +# Maps detection label IDs to the corresponding label text ("Face"). The label +# map is provided in the label_map_path option. +node { + calculator: "DetectionLabelIdToTextCalculator" + input_stream: "filtered_detections" + output_stream: "labeled_detections" + options: { + [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] { + label_map_path: "mediapipe/models/face_detection_front_labelmap.txt" + } + } +} + +# Adjusts detection locations (already normalized to [0.f, 1.f]) on the +# letterboxed image (after image transformation with the FIT scale mode) to the +# corresponding locations on the same image with the letterbox removed (the +# input image to the graph before image transformation). +node { + calculator: "DetectionLetterboxRemovalCalculator" + input_stream: "DETECTIONS:labeled_detections" + input_stream: "LETTERBOX_PADDING:letterbox_padding" + output_stream: "DETECTIONS:output_detections" +} + +# Converts the detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:output_detections" + output_stream: "RENDER_DATA:render_data" + options: { + [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] { + thickness: 4.0 + color { r: 255 g: 0 b: 0 } + } + } +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "INPUT_FRAME:throttled_input_video" + input_stream: "render_data" + output_stream: "OUTPUT_FRAME:output_video" +} + diff --git a/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt b/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt new file mode 100644 index 00000000..52de1970 --- /dev/null +++ b/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt @@ -0,0 +1,179 @@ +# MediaPipe graph that performs object detection with TensorFlow Lite on CPU. +# Used in the examples in +# mediapipie/examples/coral:object_detection_cpu. + +# Images on CPU coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for +# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish +# generating the corresponding detections before it passes through another +# image. All images that come in while waiting are dropped, limiting the number +# of in-flight images between this calculator and +# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between +# from queuing up incoming images and data excessively, which leads to increased +# latency and memory usage, unwanted in real-time mobile applications. It also +# eliminates unnecessarily computation, e.g., a transformed image produced by +# ImageTransformationCalculator may get dropped downstream if the subsequent +# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy +# processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:detections" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Transforms the input image on CPU to a 320x320 image. To scale the image, by +# default it uses the STRETCH scale mode that maps the entire input image to the +# entire transformed image. As a result, image aspect ratio may be changed and +# objects in the image may be deformed (stretched or squeezed), but the object +# detection model used in this graph is agnostic to that deformation. +node: { + calculator: "ImageTransformationCalculator" + input_stream: "IMAGE:throttled_input_video" + output_stream: "IMAGE:transformed_input_video" + options: { + [mediapipe.ImageTransformationCalculatorOptions.ext] { + output_width: 300 + output_height: 300 + } + } +} + +# Converts the transformed input image on CPU into an image tensor stored as a +# TfLiteTensor. +node { + calculator: "TfLiteConverterCalculator" + input_stream: "IMAGE:transformed_input_video" + output_stream: "TENSORS:image_tensor" + options: { + [mediapipe.TfLiteConverterCalculatorOptions.ext] { + use_quantized_tensors: true + } + } +} + +# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "TfLiteInferenceCalculator" + input_stream: "TENSORS:image_tensor" + output_stream: "TENSORS:detection_tensors" + options: { + [mediapipe.TfLiteInferenceCalculatorOptions.ext] { + model_path: "mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite" + } + } +} + +# Generates a single side packet containing a vector of SSD anchors based on +# the specification in the options. +node { + calculator: "SsdAnchorsCalculator" + output_side_packet: "anchors" + options: { + [mediapipe.SsdAnchorsCalculatorOptions.ext] { + num_layers: 6 + min_scale: 0.2 + max_scale: 0.95 + input_size_height: 300 + input_size_width: 300 + anchor_offset_x: 0.5 + anchor_offset_y: 0.5 + strides: 16 + strides: 32 + strides: 64 + strides: 128 + strides: 256 + strides: 512 + aspect_ratios: 1.0 + aspect_ratios: 2.0 + aspect_ratios: 0.5 + aspect_ratios: 3.0 + aspect_ratios: 0.3333 + reduce_boxes_in_lowest_layer: true + } + } +} + +# Decodes the detection tensors generated by the TensorFlow Lite model, based on +# the SSD anchors and the specification in the options, into a vector of +# detections. Each detection describes a detected object. +node { + calculator: "TfLiteTensorsToDetectionsCalculator" + input_stream: "TENSORS:detection_tensors" + input_side_packet: "ANCHORS:anchors" + output_stream: "DETECTIONS:detections" + options: { + [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] { + num_classes: 91 + num_boxes: 2034 + num_coords: 4 + ignore_classes: 0 + sigmoid_score: true + apply_exponential_on_box_size: true + x_scale: 10.0 + y_scale: 10.0 + h_scale: 5.0 + w_scale: 5.0 + min_score_thresh: 0.6 + } + } +} + +# Performs non-max suppression to remove excessive detections. +node { + calculator: "NonMaxSuppressionCalculator" + input_stream: "detections" + output_stream: "filtered_detections" + options: { + [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { + min_suppression_threshold: 0.4 + max_num_detections: 3 + overlap_type: INTERSECTION_OVER_UNION + return_empty_detections: true + } + } +} + +# Maps detection label IDs to the corresponding label text. The label map is +# provided in the label_map_path option. +node { + calculator: "DetectionLabelIdToTextCalculator" + input_stream: "filtered_detections" + output_stream: "output_detections" + options: { + [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] { + label_map_path: "mediapipe/examples/coral/models/object_detection_labelmap.txt" + } + } +} + +# Converts the detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:output_detections" + output_stream: "RENDER_DATA:render_data" + options: { + [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] { + thickness: 4.0 + color { r: 255 g: 0 b: 0 } + } + } +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "INPUT_FRAME:throttled_input_video" + input_stream: "render_data" + output_stream: "OUTPUT_FRAME:output_video" +} diff --git a/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite b/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite new file mode 100644 index 00000000..2f2b4b55 Binary files /dev/null and b/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite differ diff --git a/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite b/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite new file mode 100644 index 00000000..7e6cf12f Binary files /dev/null and b/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite differ diff --git a/mediapipe/examples/coral/models/object_detection_labelmap.txt b/mediapipe/examples/coral/models/object_detection_labelmap.txt new file mode 100644 index 00000000..695772dc --- /dev/null +++ b/mediapipe/examples/coral/models/object_detection_labelmap.txt @@ -0,0 +1,90 @@ +person +bicycle +car +motorcycle +airplane +bus +train +truck +boat +traffic light +fire hydrant +??? +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +??? +backpack +umbrella +??? +??? +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +??? +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +couch +potted plant +bed +??? +dining table +??? +??? +toilet +??? +tv +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +??? +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/mediapipe/examples/coral/setup.sh b/mediapipe/examples/coral/setup.sh new file mode 100755 index 00000000..d8680e49 --- /dev/null +++ b/mediapipe/examples/coral/setup.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e +set -v + +echo 'Please run this from root level mediapipe directory! \n Ex:' +echo ' sh mediapipe/examples/coral/setup.sh ' + +sleep 3 + +mkdir opencv32_arm64_libs + +cp mediapipe/examples/coral/update_sources.sh update_sources.sh +chmod +x update_sources.sh + +mv Dockerfile Dockerfile.orig +cp mediapipe/examples/coral/Dockerfile Dockerfile + +cp WORKSPACE WORKSPACE.orig +cp mediapipe/examples/coral/WORKSPACE WORKSPACE + diff --git a/mediapipe/examples/coral/update_sources.sh b/mediapipe/examples/coral/update_sources.sh new file mode 100755 index 00000000..dcd336e6 --- /dev/null +++ b/mediapipe/examples/coral/update_sources.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# To run in the Coral Docker environment. + +. /etc/os-release + +sed -i "s/deb\ /deb \[arch=amd64\]\ /g" /etc/apt/sources.list + +echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME} main universe" >> /etc/apt/sources.list +echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-updates main universe" >> /etc/apt/sources.list +echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-security main universe" >> /etc/apt/sources.list diff --git a/mediapipe/examples/desktop/README.md b/mediapipe/examples/desktop/README.md index 95e40e49..8e36e42e 100644 --- a/mediapipe/examples/desktop/README.md +++ b/mediapipe/examples/desktop/README.md @@ -9,7 +9,9 @@ bazel build -c opt mediapipe/examples/desktop/hello_world:hello_world and then run it using: ``` -bazel-bin/mediapipe/examples/desktop/hello_world/hello_world --logtostderr +export GLOG_logtostderr=1 + +bazel-bin/mediapipe/examples/desktop/hello_world/hello_world ``` **TFlite Object Detection** @@ -23,10 +25,11 @@ bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_ and run it using: ``` +export GLOG_logtostderr=1 + bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \ --calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt \ - --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file \ - --alsologtostderr + --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file ``` **TensorFlow Object Detection** @@ -34,6 +37,8 @@ bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \ To build the object detection demo using a TensorFlow model on desktop, use: ``` +export GLOG_logtostderr=1 + bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_tensorflow \ --define MEDIAPIPE_DISABLE_GPU=1 ``` @@ -41,8 +46,68 @@ bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_ and run it using: ``` +export GLOG_logtostderr=1 + bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tensorflow \ --calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt \ --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file - --alsologtostderr ``` + +**TFlite Hand Detection** + +To build the hand detection demo using a TFLite model on desktop, use: + +``` +bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1 +``` + +and run it using: + +``` +export GLOG_logtostderr=1 + +bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \ + --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt \ + --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file +``` + +**TFlite Hand Tracking** + +To build the hand tracking demo using a TFLite model on desktop, use: + +``` +bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1 +``` + +and run it using: + +``` +export GLOG_logtostderr=1 + +bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \ + --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt \ + --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file +``` + +**TFlite Multi-Hand Tracking** + +To build the multi-hand tracking demo using a TFLite model on desktop, use: + +``` +bazel build -c opt mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1 +``` + +and run it using: + +``` +export GLOG_logtostderr=1 + +bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_tflite \ + --calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt \ + --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file +``` + +To change the number of hands to `x` in this application, change: + +1. `min_size:x` in `CollectionHasMinSizeCalculatorOptions` in `mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt`. +2. `max_vec_size:x` in `ClipVectorSizeCalculatorOptions` in `mediapipe/examples/dekstop/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt`. diff --git a/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py b/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py index 1d7f5b69..93c62d54 100644 --- a/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py +++ b/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py @@ -93,15 +93,15 @@ FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow" SPLITS = { "train": { "shards": 1000, - "examples": 541632 + "examples": 541490 }, "validate": { "shards": 100, - "examples": 34727 + "examples": 34715 }, "test": { "shards": 100, - "examples": 69347 + "examples": 69321 }, "custom": { "csv": None, # Add a CSV for your own data here. diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index d5586ae9..d9471d94 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -1524,7 +1524,9 @@ cc_test( "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", "//mediapipe/framework/stream_handler:immediate_input_stream_handler", "//mediapipe/framework/stream_handler:mux_input_stream_handler", + "//mediapipe/framework/stream_handler:sync_set_input_stream_handler", "//mediapipe/framework/tool:sink", + "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/framework/calculator_graph_bounds_test.cc b/mediapipe/framework/calculator_graph_bounds_test.cc index b6144c0a..17998a1f 100644 --- a/mediapipe/framework/calculator_graph_bounds_test.cc +++ b/mediapipe/framework/calculator_graph_bounds_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/strings/str_replace.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/canonical_errors.h" @@ -642,6 +643,36 @@ REGISTER_CALCULATOR(OffsetBoundCalculator); // A Calculator that produces a packet for each call to Process. class BoundToPacketCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + for (int i = 0; i < cc->Inputs().NumEntries(); ++i) { + cc->Inputs().Index(i).SetAny(); + } + for (int i = 0; i < cc->Outputs().NumEntries(); ++i) { + cc->Outputs().Index(i).Set(); + } + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) final { + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) final { + for (int i = 0; i < cc->Outputs().NumEntries(); ++i) { + Timestamp t = cc->Inputs().Index(i).Value().Timestamp(); + cc->Outputs().Index(i).AddPacket( + mediapipe::MakePacket(t).At(cc->InputTimestamp())); + } + return ::mediapipe::OkStatus(); + } +}; +REGISTER_CALCULATOR(BoundToPacketCalculator); + +// A Calculator that produces packets at timestamps beyond the input timestamp. +class FuturePacketCalculator : public CalculatorBase { + static constexpr int64 kOutputFutureMicros = 3; + public: static ::mediapipe::Status GetContract(CalculatorContract* cc) { cc->Inputs().Index(0).Set(); @@ -654,11 +685,14 @@ class BoundToPacketCalculator : public CalculatorBase { } ::mediapipe::Status Process(CalculatorContext* cc) final { - cc->Outputs().Index(0).AddPacket(Adopt(new int(33))); + const Packet& packet = cc->Inputs().Index(0).Value(); + Timestamp timestamp = + Timestamp(packet.Timestamp().Value() + kOutputFutureMicros); + cc->Outputs().Index(0).AddPacket(packet.At(timestamp)); return ::mediapipe::OkStatus(); } }; -REGISTER_CALCULATOR(BoundToPacketCalculator); +REGISTER_CALCULATOR(FuturePacketCalculator); // Verifies that SetOffset still propagates when Process is called and // produces no output packets. @@ -964,5 +998,111 @@ TEST(CalculatorGraphBoundsTest, LastPacketCheck) { MP_ASSERT_OK(graph.WaitUntilDone()); } +// Shows that bounds are indicated for input streams without input packets. +void TestBoundsForEmptyInputs(std::string input_stream_handler) { + // FuturePacketCalculator and OffsetBoundCalculator produce future ts bounds. + // BoundToPacketCalculator reports all of its bounds, including empty inputs. + std::string config_str = R"( + input_stream: 'input' + node { + calculator: 'FuturePacketCalculator' + input_stream: 'input' + output_stream: 'futures' + } + node { + calculator: 'OffsetBoundCalculator' + input_stream: 'futures' + output_stream: 'bounds' + } + node { + calculator: 'BoundToPacketCalculator' + input_stream: 'input' + input_stream: 'bounds' + output_stream: 'input_ts' + output_stream: 'bounds_ts' + input_stream_handler { $input_stream_handler } + } + )"; + absl::StrReplaceAll({{"$input_stream_handler", input_stream_handler}}, + &config_str); + CalculatorGraphConfig config = + ::mediapipe::ParseTextProtoOrDie(config_str); + CalculatorGraph graph; + std::vector input_ts_packets; + std::vector bounds_ts_packets; + MP_ASSERT_OK(graph.Initialize(config)); + MP_ASSERT_OK(graph.ObserveOutputStream("input_ts", [&](const Packet& p) { + input_ts_packets.push_back(p); + return ::mediapipe::OkStatus(); + })); + MP_ASSERT_OK(graph.ObserveOutputStream("bounds_ts", [&](const Packet& p) { + bounds_ts_packets.push_back(p); + return ::mediapipe::OkStatus(); + })); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + + // Add four packets into the graph, with timedtamps 0, 10, 20, 30. + constexpr int kNumInputs = 4; + for (int i = 0; i < kNumInputs; ++i) { + Packet p = MakePacket(33).At(Timestamp(i * 10)); + MP_ASSERT_OK(graph.AddPacketToInputStream("input", p)); + MP_ASSERT_OK(graph.WaitUntilIdle()); + } + + // Packets arrive. The input packet timestamps are: 0, 10, 20, 30. + // The corresponding empty packet timestamps are: 3, 13, 23, 33. + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_EQ(input_ts_packets.size(), 4); + EXPECT_EQ(bounds_ts_packets.size(), 4); + + // The timestamp bounds from OffsetBoundCalculator are: 3, 13, 23, 33. + // Because the process call waits for the input packets and not for + // the empty packets, the first empty packet timestamp can be + // either Timestamp::Unstarted() or Timestamp(3). + std::vector expected = {Timestamp::Unstarted(), Timestamp(3), + Timestamp(13), Timestamp(23), + Timestamp(33)}; + for (int i = 0; i < bounds_ts_packets.size(); ++i) { + Timestamp ts = bounds_ts_packets[i].Get(); + EXPECT_GE(ts, expected[i]); + EXPECT_LE(ts, expected[i + 1]); + } + + // Shutdown the graph. + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + +// Shows that bounds are indicated for input streams without input packets. +TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_Immediate) { + TestBoundsForEmptyInputs(R"( + input_stream_handler: "ImmediateInputStreamHandler")"); +} + +// Shows that bounds are indicated for input streams without input packets. +TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_Default) { + TestBoundsForEmptyInputs(R"( + input_stream_handler: "DefaultInputStreamHandler")"); +} + +// Shows that bounds are indicated for input streams without input packets. +TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_SyncSet) { + TestBoundsForEmptyInputs(R"( + input_stream_handler: "SyncSetInputStreamHandler")"); +} + +// Shows that bounds are indicated for input streams without input packets. +TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_SyncSets) { + TestBoundsForEmptyInputs(R"( + input_stream_handler: "SyncSetInputStreamHandler" + options { + [mediapipe.SyncSetInputStreamHandlerOptions.ext] { + sync_set { tag_index: ":0" } + } + } + )"); +} + } // namespace } // namespace mediapipe diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index c7ab46bb..231fd321 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -27,72 +27,67 @@ package( proto_library( name = "detection_proto", srcs = ["detection.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = ["//mediapipe/framework/formats:location_data_proto"], ) proto_library( name = "classification_proto", srcs = ["classification.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], ) proto_library( name = "image_format_proto", srcs = ["image_format.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], ) proto_library( name = "matrix_data_proto", srcs = ["matrix_data.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], ) proto_library( name = "location_data_proto", srcs = ["location_data.proto"], + visibility = ["//visibility:public"], deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"], ) proto_library( name = "time_series_header_proto", srcs = ["time_series_header.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], ) mediapipe_cc_proto_library( name = "detection_cc_proto", srcs = ["detection.proto"], cc_deps = [":location_data_cc_proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":detection_proto"], ) mediapipe_cc_proto_library( name = "classification_cc_proto", srcs = ["classification.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":classification_proto"], ) mediapipe_cc_proto_library( name = "image_format_cc_proto", srcs = ["image_format.proto"], - visibility = [ - "//mediapipe:__subpackages__", - "//mediapipe/java/com/google/mediapipe/framework:__subpackages__", - ], + visibility = ["//visibility:public"], deps = [":image_format_proto"], ) mediapipe_cc_proto_library( name = "matrix_data_cc_proto", srcs = ["matrix_data.proto"], - visibility = [ - "//mediapipe:__subpackages__", - "//mediapipe/java/com/google/mediapipe/framework:__subpackages__", - ], + visibility = ["//visibility:public"], deps = [":matrix_data_proto"], ) @@ -100,17 +95,14 @@ mediapipe_cc_proto_library( name = "location_data_cc_proto", srcs = ["location_data.proto"], cc_deps = ["//mediapipe/framework/formats/annotation:rasterization_cc_proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":location_data_proto"], ) mediapipe_cc_proto_library( name = "time_series_header_cc_proto", srcs = ["time_series_header.proto"], - visibility = [ - "//mediapipe:__subpackages__", - "//mediapipe/java/com/google/mediapipe/framework:__subpackages__", - ], + visibility = ["//visibility:public"], deps = [":time_series_header_proto"], ) @@ -249,14 +241,14 @@ cc_test( proto_library( name = "rect_proto", srcs = ["rect.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], ) mediapipe_cc_proto_library( name = "rect_cc_proto", srcs = ["rect.proto"], visibility = [ - "//mediapipe:__subpackages__", + "//visibility:public", ], deps = [":rect_proto"], ) @@ -264,12 +256,12 @@ mediapipe_cc_proto_library( proto_library( name = "landmark_proto", srcs = ["landmark.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], ) mediapipe_cc_proto_library( name = "landmark_cc_proto", srcs = ["landmark.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":landmark_proto"], ) diff --git a/mediapipe/framework/formats/annotation/BUILD b/mediapipe/framework/formats/annotation/BUILD index 939e4d95..5ea495ab 100644 --- a/mediapipe/framework/formats/annotation/BUILD +++ b/mediapipe/framework/formats/annotation/BUILD @@ -25,26 +25,27 @@ package(default_visibility = ["//visibility:private"]) proto_library( name = "locus_proto", srcs = ["locus.proto"], + visibility = ["//visibility:public"], deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"], ) proto_library( name = "rasterization_proto", srcs = ["rasterization.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], ) mediapipe_cc_proto_library( name = "locus_cc_proto", srcs = ["locus.proto"], cc_deps = [":rasterization_cc_proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":locus_proto"], ) mediapipe_cc_proto_library( name = "rasterization_cc_proto", srcs = ["rasterization.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":rasterization_proto"], ) diff --git a/mediapipe/framework/formats/landmark.proto b/mediapipe/framework/formats/landmark.proto index 220b3725..cfafb793 100644 --- a/mediapipe/framework/formats/landmark.proto +++ b/mediapipe/framework/formats/landmark.proto @@ -25,6 +25,11 @@ message Landmark { optional float z = 3; } +// Group of Landmark protos. +message LandmarkList { + repeated Landmark landmark = 1; +} + // A normalized version of above Landmark proto. All coordiates should be within // [0, 1]. message NormalizedLandmark { diff --git a/mediapipe/framework/formats/motion/BUILD b/mediapipe/framework/formats/motion/BUILD index c065eae0..a77034c5 100644 --- a/mediapipe/framework/formats/motion/BUILD +++ b/mediapipe/framework/formats/motion/BUILD @@ -27,12 +27,13 @@ package(default_visibility = ["//visibility:private"]) proto_library( name = "optical_flow_field_data_proto", srcs = ["optical_flow_field_data.proto"], + visibility = ["//visibility:public"], ) mediapipe_cc_proto_library( name = "optical_flow_field_data_cc_proto", srcs = ["optical_flow_field_data.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":optical_flow_field_data_proto"], ) @@ -41,7 +42,7 @@ cc_library( srcs = ["optical_flow_field.cc"], hdrs = ["optical_flow_field.h"], visibility = [ - "//mediapipe:__subpackages__", + "//visibility:public", ], deps = [ "//mediapipe/framework:type_map", diff --git a/mediapipe/framework/formats/object_detection/BUILD b/mediapipe/framework/formats/object_detection/BUILD index 060281b6..c073afa0 100644 --- a/mediapipe/framework/formats/object_detection/BUILD +++ b/mediapipe/framework/formats/object_detection/BUILD @@ -24,12 +24,12 @@ package(default_visibility = ["//visibility:private"]) proto_library( name = "anchor_proto", srcs = ["anchor.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], ) mediapipe_cc_proto_library( name = "anchor_cc_proto", srcs = ["anchor.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":anchor_proto"], ) diff --git a/mediapipe/framework/input_stream_manager.cc b/mediapipe/framework/input_stream_manager.cc index edbe6a68..70d67a55 100644 --- a/mediapipe/framework/input_stream_manager.cc +++ b/mediapipe/framework/input_stream_manager.cc @@ -229,6 +229,11 @@ Timestamp InputStreamManager::MinTimestampOrBound(bool* is_empty) const { if (is_empty) { *is_empty = queue_.empty(); } + return MinTimestampOrBoundHelper(); +} + +Timestamp InputStreamManager::MinTimestampOrBoundHelper() const + EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_) { return queue_.empty() ? next_timestamp_bound_ : queue_.front().Timestamp(); } @@ -271,7 +276,9 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp, } // Clear value_ if it doesn't have exactly the right timestamp. if (current_timestamp != timestamp) { - packet = Packet(); + // The timestamp bound reported when no packet is sent. + Timestamp bound = MinTimestampOrBoundHelper(); + packet = Packet().At(bound.PreviousAllowedInStream()); ++(*num_packets_dropped); } diff --git a/mediapipe/framework/input_stream_manager.h b/mediapipe/framework/input_stream_manager.h index c0a37403..48dfcf36 100644 --- a/mediapipe/framework/input_stream_manager.h +++ b/mediapipe/framework/input_stream_manager.h @@ -189,6 +189,9 @@ class InputStreamManager { // Returns true if the next timestamp bound reaches Timestamp::Done(). bool IsDone() const EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_); + // Returns the smallest timestamp at which this stream might see an input. + Timestamp MinTimestampOrBoundHelper() const; + mutable absl::Mutex stream_mutex_; std::deque queue_ GUARDED_BY(stream_mutex_); // The number of packets added to queue_. Used to verify a packet at diff --git a/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc b/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc index 8dd5c53d..97b0ad78 100644 --- a/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc @@ -115,9 +115,10 @@ void ImmediateInputStreamHandler::FillInputSet(Timestamp input_timestamp, AddPacketToShard(&input_set->Get(id), std::move(current_packet), stream_is_done); } else { - bool empty = false; - bool is_done = stream->MinTimestampOrBound(&empty) == Timestamp::Done(); - AddPacketToShard(&input_set->Get(id), Packet(), is_done); + Timestamp bound = stream->MinTimestampOrBound(nullptr); + AddPacketToShard(&input_set->Get(id), + Packet().At(bound.PreviousAllowedInStream()), + bound == Timestamp::Done()); } } } diff --git a/mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc b/mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc index 26bfe5ae..4175a767 100644 --- a/mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc @@ -56,9 +56,16 @@ class SyncSetInputStreamHandler : public InputStreamHandler { NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. + // Populates packets for the ready sync-set, and populates timestamp bounds + // for all sync-sets. void FillInputSet(Timestamp input_timestamp, InputStreamShardSet* input_set) override; + // Populates timestamp bounds for streams outside the ready sync-set. + void FillInputBounds(Timestamp input_timestamp, + InputStreamShardSet* input_set) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + private: absl::Mutex mutex_; // The ids of each set of inputs. @@ -181,6 +188,22 @@ NodeReadiness SyncSetInputStreamHandler::GetNodeReadiness( return NodeReadiness::kNotReady; } +void SyncSetInputStreamHandler::FillInputBounds( + Timestamp input_timestamp, InputStreamShardSet* input_set) { + for (int i = 0; i < sync_sets_.size(); ++i) { + if (i != ready_sync_set_index_) { + // Set the input streams for the not-ready sync sets. + for (CollectionItemId id : sync_sets_[i]) { + const auto stream = input_stream_managers_.Get(id); + Timestamp bound = stream->MinTimestampOrBound(nullptr); + AddPacketToShard(&input_set->Get(id), + Packet().At(bound.PreviousAllowedInStream()), + bound == Timestamp::Done()); + } + } + } +} + void SyncSetInputStreamHandler::FillInputSet(Timestamp input_timestamp, InputStreamShardSet* input_set) { // Assume that all current packets are already cleared. @@ -202,6 +225,7 @@ void SyncSetInputStreamHandler::FillInputSet(Timestamp input_timestamp, AddPacketToShard(&input_set->Get(id), std::move(current_packet), stream_is_done); } + FillInputBounds(input_timestamp, input_set); ready_sync_set_index_ = -1; ready_timestamp_ = Timestamp::Done(); } diff --git a/mediapipe/framework/timestamp.cc b/mediapipe/framework/timestamp.cc index bacbc3f4..99d48b14 100644 --- a/mediapipe/framework/timestamp.cc +++ b/mediapipe/framework/timestamp.cc @@ -123,13 +123,23 @@ std::string TimestampDiff::DebugString() const { Timestamp Timestamp::NextAllowedInStream() const { CHECK(IsAllowedInStream()) << "Timestamp is: " << DebugString(); - if (IsRangeValue() && *this != Max()) { - return *this + 1; - } else { - // Returning this value effectively means no futher timestamps may - // occur (however, the stream is not yet closed). + if (*this >= Max() || *this == PreStream()) { + // Indicates that no further timestamps may occur. return OneOverPostStream(); + } else if (*this < Min()) { + return Min(); } + return *this + 1; +} + +Timestamp Timestamp::PreviousAllowedInStream() const { + if (*this <= Min() || *this == PostStream()) { + // Indicates that no previous timestamps may occur. + return Unstarted(); + } else if (*this > Max()) { + return Max(); + } + return *this - 1; } std::ostream& operator<<(std::ostream& os, Timestamp arg) { diff --git a/mediapipe/framework/timestamp.h b/mediapipe/framework/timestamp.h index d6525c5c..a79c4fd7 100644 --- a/mediapipe/framework/timestamp.h +++ b/mediapipe/framework/timestamp.h @@ -182,12 +182,15 @@ class Timestamp { Timestamp operator++(int); Timestamp operator--(int); - // Returns the next timestamp at which a Packet may arrive in a stream, given - // that the current Packet is at *this timestamp. CHECKs that - // this->IsAllowedInStream()==true. Returns Timestamp::OneOverPostStream() if - // no Packets may follow one with the given timestamp. + // Returns the next timestamp in the range [Min .. Max], or + // OneOverPostStream() if no Packets may follow one with this timestamp. + // CHECKs that this->IsAllowedInStream(). Timestamp NextAllowedInStream() const; + // Returns the previous timestamp in the range [Min .. Max], or + // Unstarted() if no Packets may preceed one with this timestamp. + Timestamp PreviousAllowedInStream() const; + private: TimestampBaseType timestamp_; }; diff --git a/mediapipe/gpu/metal.bzl b/mediapipe/gpu/metal.bzl index 2b45c661..0f19ec2c 100644 --- a/mediapipe/gpu/metal.bzl +++ b/mediapipe/gpu/metal.bzl @@ -132,7 +132,7 @@ def _metal_library_impl(ctx): ), ) - # This ridiculous circumlocution is needed because new_objc_provider rejects + # This circumlocution is needed because new_objc_provider rejects # an empty depset, with the error: # "Value for key header must be a set of File, instead found set of unknown." # It also rejects an explicit "None". diff --git a/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt b/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt index 1a554629..aa4b4ae4 100644 --- a/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt +++ b/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt @@ -94,7 +94,7 @@ node { output_stream: "multi_hand_rects" node_options: { [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { - min_similarity_threshold: 0.1 + min_similarity_threshold: 0.5 } } } diff --git a/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt b/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt index 6e4a0331..f009f49c 100644 --- a/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt +++ b/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt @@ -85,7 +85,7 @@ node { output_stream: "multi_hand_rects" node_options: { [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { - min_similarity_threshold: 0.1 + min_similarity_threshold: 0.5 } } } diff --git a/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt b/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt index c47bc3d8..87f65117 100644 --- a/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt +++ b/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt @@ -105,7 +105,7 @@ node { output_stream: "multi_hand_rects" node_options: { [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { - min_similarity_threshold: 0.1 + min_similarity_threshold: 0.5 } } } diff --git a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt index aa055731..928e7521 100644 --- a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt +++ b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt @@ -104,7 +104,7 @@ node { y_scale: 256.0 h_scale: 256.0 w_scale: 256.0 - min_score_thresh: 0.5 + min_score_thresh: 0.7 } } } @@ -117,7 +117,6 @@ node { node_options: { [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { min_suppression_threshold: 0.3 - min_score_threshold: 0.5 overlap_type: INTERSECTION_OVER_UNION algorithm: WEIGHTED return_empty_detections: true diff --git a/mediapipe/util/audio_decoder.cc b/mediapipe/util/audio_decoder.cc index 4f284f10..b101e1c3 100644 --- a/mediapipe/util/audio_decoder.cc +++ b/mediapipe/util/audio_decoder.cc @@ -340,9 +340,7 @@ AudioPacketProcessor::AudioPacketProcessor(const AudioStreamOptions& options) DCHECK(absl::little_endian::IsLittleEndian()); } -mediapipe::Status AudioPacketProcessor::Open(int id, - - AVStream* stream) { +mediapipe::Status AudioPacketProcessor::Open(int id, AVStream* stream) { id_ = id; avcodec_ = avcodec_find_decoder(stream->codecpar->codec_id); if (!avcodec_) { diff --git a/mediapipe/util/sequence/media_sequence.cc b/mediapipe/util/sequence/media_sequence.cc index d50055da..9f00be3c 100644 --- a/mediapipe/util/sequence/media_sequence.cc +++ b/mediapipe/util/sequence/media_sequence.cc @@ -274,10 +274,11 @@ float TimestampsToRate(int64 first_timestamp, int64 second_timestamp) { // overwriting with modified values. if (!GetUnmodifiedBBoxTimestampSize(prefix, *sequence)) { for (int i = 0; i < num_frames; ++i) { - if (bbox_index_if_annotated[i] >= 0 && - GetBBoxIsAnnotatedAt(prefix, *sequence, i)) { - AddUnmodifiedBBoxTimestamp( - prefix, box_timestamps[bbox_index_if_annotated[i]], sequence); + const int bbox_index = bbox_index_if_annotated[i]; + if (bbox_index >= 0 && + GetBBoxIsAnnotatedAt(prefix, *sequence, bbox_index)) { + AddUnmodifiedBBoxTimestamp(prefix, box_timestamps[bbox_index], + sequence); } } } diff --git a/mediapipe/util/sequence/media_sequence_test.cc b/mediapipe/util/sequence/media_sequence_test.cc index 26bfb6fc..93b3267b 100644 --- a/mediapipe/util/sequence/media_sequence_test.cc +++ b/mediapipe/util/sequence/media_sequence_test.cc @@ -738,10 +738,11 @@ TEST(MediaSequenceTest, AddImageTimestamp(10, &sequence); AddImageTimestamp(20, &sequence); AddImageTimestamp(30, &sequence); + AddImageTimestamp(40, &sequence); - AddBBoxTimestamp(9, &sequence); - AddBBoxTimestamp(21, &sequence); - AddBBoxTimestamp(22, &sequence); // Will be dropped in the output. + AddBBoxTimestamp(11, &sequence); + AddBBoxTimestamp(12, &sequence); // Will be dropped in the output. + AddBBoxTimestamp(39, &sequence); std::vector> bboxes = { {Location::CreateRelativeBBoxLocation(0.1, 0.2, 0.7, 0.7)}, @@ -753,32 +754,35 @@ TEST(MediaSequenceTest, MP_ASSERT_OK(ReconcileMetadata(true, false, &sequence)); - ASSERT_EQ(GetBBoxTimestampSize(sequence), 3); + ASSERT_EQ(GetBBoxTimestampSize(sequence), 4); ASSERT_EQ(GetBBoxTimestampAt(sequence, 0), 10); ASSERT_EQ(GetBBoxTimestampAt(sequence, 1), 20); ASSERT_EQ(GetBBoxTimestampAt(sequence, 2), 30); + ASSERT_EQ(GetBBoxTimestampAt(sequence, 3), 40); - ASSERT_EQ(GetBBoxIsAnnotatedSize(sequence), 3); + ASSERT_EQ(GetBBoxIsAnnotatedSize(sequence), 4); ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 0), true); - ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 1), true); + ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 1), false); ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 2), false); + ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 3), true); // Unmodified timestamp is only stored for is_annotated == true. ASSERT_EQ(GetUnmodifiedBBoxTimestampSize(sequence), 2); - ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 9); - ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 21); + ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 11); + ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 39); // A second reconciliation should not corrupt unmodified bbox timestamps. MP_ASSERT_OK(ReconcileMetadata(true, false, &sequence)); - ASSERT_EQ(GetBBoxTimestampSize(sequence), 3); + ASSERT_EQ(GetBBoxTimestampSize(sequence), 4); ASSERT_EQ(GetBBoxTimestampAt(sequence, 0), 10); ASSERT_EQ(GetBBoxTimestampAt(sequence, 1), 20); ASSERT_EQ(GetBBoxTimestampAt(sequence, 2), 30); + ASSERT_EQ(GetBBoxTimestampAt(sequence, 3), 40); ASSERT_EQ(GetUnmodifiedBBoxTimestampSize(sequence), 2); - ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 9); - ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 21); + ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 11); + ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 39); } TEST(MediaSequenceTest, ReconcileMetadataBoxAnnotationsFillsMissing) { diff --git a/third_party/BUILD b/third_party/BUILD index 65fda9d2..4c74553b 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -118,13 +118,16 @@ android_library( ], ) -# TODO: Get the AARs from Google's Maven Repository. -aar_import( +android_library( name = "camerax_core", - aar = "camera-core-1.0.0-alpha01.aar", + exports = [ + "@maven//:androidx_camera_camera_core", + ], ) -aar_import( +android_library( name = "camera2", - aar = "camera-camera2-1.0.0-alpha01.aar", + exports = [ + "@maven//:androidx_camera_camera_camera2", + ], ) diff --git a/third_party/camera-camera2-1.0.0-alpha01.aar b/third_party/camera-camera2-1.0.0-alpha01.aar deleted file mode 100644 index ee89bb96..00000000 Binary files a/third_party/camera-camera2-1.0.0-alpha01.aar and /dev/null differ diff --git a/third_party/camera-core-1.0.0-alpha01.aar b/third_party/camera-core-1.0.0-alpha01.aar deleted file mode 100644 index 593c3bad..00000000 Binary files a/third_party/camera-core-1.0.0-alpha01.aar and /dev/null differ diff --git a/third_party/opencv_android.BUILD b/third_party/opencv_android.BUILD index e0e6bd60..3bdfc88d 100644 --- a/third_party/opencv_android.BUILD +++ b/third_party/opencv_android.BUILD @@ -5,7 +5,7 @@ licenses(["notice"]) # BSD license exports_files(["LICENSE"]) -OPENCV_LIBRARY_NAME = "libopencv_java4.so" +OPENCV_LIBRARY_NAME = "libopencv_java3.so" OPENCVANDROIDSDK_NATIVELIBS_PATH = "sdk/native/libs/"