Project import generated by Copybara.
GitOrigin-RevId: d073f8e21be2fcc0e503cb97c6695078b6b75310
This commit is contained in:
MediaPipe Team
@@ -67,26 +67,26 @@ class CalculatorBase {
|
||||
|
||||
// The subclasses of CalculatorBase must implement GetContract.
|
||||
// ...
|
||||
static ::MediaPipe::Status GetContract(CalculatorContract* cc);
|
||||
static absl::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
// Open is called before any Process() calls, on a freshly constructed
|
||||
// calculator. Subclasses may override this method to perform necessary
|
||||
// setup, and possibly output Packets and/or set output streams' headers.
|
||||
// ...
|
||||
virtual ::MediaPipe::Status Open(CalculatorContext* cc) {
|
||||
return ::MediaPipe::OkStatus();
|
||||
virtual absl::Status Open(CalculatorContext* cc) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
// Processes the incoming inputs. May call the methods on cc to access
|
||||
// inputs and produce outputs.
|
||||
// ...
|
||||
virtual ::MediaPipe::Status Process(CalculatorContext* cc) = 0;
|
||||
virtual absl::Status Process(CalculatorContext* cc) = 0;
|
||||
|
||||
// Is called if Open() was called and succeeded. Is called either
|
||||
// immediately after processing is complete or after a graph run has ended
|
||||
// (if an error occurred in the graph). ...
|
||||
virtual ::MediaPipe::Status Close(CalculatorContext* cc) {
|
||||
return ::MediaPipe::OkStatus();
|
||||
virtual absl::Status Close(CalculatorContext* cc) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
...
|
||||
@@ -199,7 +199,7 @@ name and index number. In the function below input are output are identified:
|
||||
// c++ Code snippet describing the SomeAudioVideoCalculator GetContract() method
|
||||
class SomeAudioVideoCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).SetAny();
|
||||
// SetAny() is used to specify that whatever the type of the
|
||||
// stream is, it's acceptable. This does not mean that any
|
||||
@@ -209,13 +209,13 @@ class SomeAudioVideoCalculator : public CalculatorBase {
|
||||
cc->Outputs().Tag("VIDEO").Set<ImageFrame>();
|
||||
cc->Outputs().Get("AUDIO", 0).Set<Matrix>();
|
||||
cc->Outputs().Get("AUDIO", 1).Set<Matrix>();
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
```
|
||||
|
||||
## Processing
|
||||
|
||||
`Process()` called on a non-source node must return `::mediapipe::OkStatus()` to
|
||||
`Process()` called on a non-source node must return `absl::OkStatus()` to
|
||||
indicate that all went well, or any other status code to signal an error
|
||||
|
||||
If a non-source calculator returns `tool::StatusStop()`, then this signals the
|
||||
@@ -224,12 +224,12 @@ input streams will be closed (and remaining Packets will propagate through the
|
||||
graph).
|
||||
|
||||
A source node in a graph will continue to have `Process()` called on it as long
|
||||
as it returns `::mediapipe::OkStatus(`). To indicate that there is no more data
|
||||
to be generated return `tool::StatusStop()`. Any other status indicates an error
|
||||
has occurred.
|
||||
as it returns `absl::OkStatus(`). To indicate that there is no more data to be
|
||||
generated return `tool::StatusStop()`. Any other status indicates an error has
|
||||
occurred.
|
||||
|
||||
`Close()` returns `::mediapipe::OkStatus()` to indicate success. Any other
|
||||
status indicates a failure.
|
||||
`Close()` returns `absl::OkStatus()` to indicate success. Any other status
|
||||
indicates a failure.
|
||||
|
||||
Here is the basic `Process()` function. It uses the `Input()` method (which can
|
||||
be used only if the calculator has a single input) to request its input data. It
|
||||
@@ -238,13 +238,13 @@ and does the calculations. When done it releases the pointer when adding it to
|
||||
the output stream.
|
||||
|
||||
```c++
|
||||
::util::Status MyCalculator::Process() {
|
||||
absl::Status MyCalculator::Process() {
|
||||
const Matrix& input = Input()->Get<Matrix>();
|
||||
std::unique_ptr<Matrix> output(new Matrix(input.rows(), input.cols()));
|
||||
// do your magic here....
|
||||
// output->row(n) = ...
|
||||
Output()->Add(output.release(), InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
```
|
||||
|
||||
@@ -312,7 +312,7 @@ namespace mediapipe {
|
||||
//
|
||||
class PacketClonerCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
static absl::Status GetContract(CalculatorContract* cc) {
|
||||
const int tick_signal_index = cc->Inputs().NumEntries() - 1;
|
||||
// cc->Inputs().NumEntries() returns the number of input streams
|
||||
// for the PacketClonerCalculator
|
||||
@@ -322,10 +322,10 @@ class PacketClonerCalculator : public CalculatorBase {
|
||||
cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
|
||||
}
|
||||
cc->Inputs().Index(tick_signal_index).SetAny();
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) final {
|
||||
absl::Status Open(CalculatorContext* cc) final {
|
||||
tick_signal_index_ = cc->Inputs().NumEntries() - 1;
|
||||
current_.resize(tick_signal_index_);
|
||||
// Pass along the header for each stream if present.
|
||||
@@ -336,10 +336,10 @@ class PacketClonerCalculator : public CalculatorBase {
|
||||
// the header for the input stream of index i
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) final {
|
||||
absl::Status Process(CalculatorContext* cc) final {
|
||||
// Store input signals.
|
||||
for (int i = 0; i < tick_signal_index_; ++i) {
|
||||
if (!cc->Inputs().Index(i).Value().IsEmpty()) {
|
||||
@@ -364,7 +364,7 @@ class PacketClonerCalculator : public CalculatorBase {
|
||||
}
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@@ -66,10 +66,10 @@ calculator derived from base class GlSimpleCalculator. The GPU calculator
|
||||
// See GlSimpleCalculator for inputs, outputs and input side packets.
|
||||
class LuminanceCalculator : public GlSimpleCalculator {
|
||||
public:
|
||||
::mediapipe::Status GlSetup() override;
|
||||
::mediapipe::Status GlRender(const GlTexture& src,
|
||||
const GlTexture& dst) override;
|
||||
::mediapipe::Status GlTeardown() override;
|
||||
absl::Status GlSetup() override;
|
||||
absl::Status GlRender(const GlTexture& src,
|
||||
const GlTexture& dst) override;
|
||||
absl::Status GlTeardown() override;
|
||||
|
||||
private:
|
||||
GLuint program_ = 0;
|
||||
@@ -77,8 +77,8 @@ class LuminanceCalculator : public GlSimpleCalculator {
|
||||
};
|
||||
REGISTER_CALCULATOR(LuminanceCalculator);
|
||||
|
||||
::mediapipe::Status LuminanceCalculator::GlRender(const GlTexture& src,
|
||||
const GlTexture& dst) {
|
||||
absl::Status LuminanceCalculator::GlRender(const GlTexture& src,
|
||||
const GlTexture& dst) {
|
||||
static const GLfloat square_vertices[] = {
|
||||
-1.0f, -1.0f, // bottom left
|
||||
1.0f, -1.0f, // bottom right
|
||||
@@ -128,7 +128,7 @@ REGISTER_CALCULATOR(LuminanceCalculator);
|
||||
glDeleteVertexArrays(1, &vao);
|
||||
glDeleteBuffers(2, vbo);
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -219,23 +219,23 @@ packet timestamps 0, 1, 2, 3, ...
|
||||
```c++
|
||||
class UnitDelayCalculator : public Calculator {
|
||||
public:
|
||||
static ::util::Status FillExpectations(
|
||||
static absl::Status FillExpectations(
|
||||
const CalculatorOptions& extendable_options, PacketTypeSet* inputs,
|
||||
PacketTypeSet* outputs, PacketTypeSet* input_side_packets) {
|
||||
inputs->Index(0)->Set<int>("An integer.");
|
||||
outputs->Index(0)->Set<int>("The input delayed by one time unit.");
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
::util::Status Open() final {
|
||||
absl::Status Open() final {
|
||||
Output()->Add(new int(0), Timestamp(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
::util::Status Process() final {
|
||||
absl::Status Process() final {
|
||||
const Packet& packet = Input()->Value();
|
||||
Output()->AddPacket(packet.At(packet.Timestamp().NextAllowedInStream()));
|
||||
return ::mediapipe::OkStatus();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
@@ -45,7 +45,8 @@ each project.
|
||||
2. Run the Bazel build command to generate the AAR.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
|
||||
bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
|
||||
--fat_apk_cpu=arm64-v8a,armeabi-v7a --strip=ALWAYS \
|
||||
//path/to/the/aar/build/file:aar_name
|
||||
```
|
||||
|
||||
@@ -86,16 +87,14 @@ each project.
|
||||
Build the MediaPipe binary graph and copy the assets into
|
||||
app/src/main/assets, e.g., for the face detection graph, you need to build
|
||||
and copy
|
||||
[the binary graph](https://github.com/google/mediapipe/blob/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD#L41),
|
||||
[the tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
|
||||
[the binary graph](https://github.com/google/mediapipe/blob/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD#L41)
|
||||
and
|
||||
[the label map](https://github.com/google/mediapipe/blob/master/mediapipe/models/face_detection_front_labelmap.txt).
|
||||
[the face detection tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite).
|
||||
|
||||
```bash
|
||||
bazel build -c opt mediapipe/mediapipe/graphs/face_detection:mobile_gpu_binary_graph
|
||||
cp bazel-bin/mediapipe/graphs/face_detection/mobile_gpu.binarypb /path/to/your/app/src/main/assets/
|
||||
cp mediapipe/models/face_detection_front.tflite /path/to/your/app/src/main/assets/
|
||||
cp mediapipe/models/face_detection_front_labelmap.txt /path/to/your/app/src/main/assets/
|
||||
cp mediapipe/modules/face_detection/face_detection_front.tflite /path/to/your/app/src/main/assets/
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ node: {
|
||||
output_stream: "luma_video"
|
||||
}
|
||||
|
||||
# Applies the Sobel filter to luminance images sotred in RGB format.
|
||||
# Applies the Sobel filter to luminance images stored in RGB format.
|
||||
node: {
|
||||
calculator: "SobelEdgesCalculator"
|
||||
input_stream: "luma_video"
|
||||
|
||||
@@ -44,7 +44,7 @@ nav_order: 1
|
||||
`PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto.
|
||||
|
||||
```C++
|
||||
::mediapipe::Status PrintHelloWorld() {
|
||||
absl::Status PrintHelloWorld() {
|
||||
// Configures a simple graph, which concatenates 2 PassThroughCalculators.
|
||||
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||
input_stream: "in"
|
||||
|
||||
@@ -492,6 +492,9 @@ in our app:
|
||||
if (![self.mediapipeGraph startWithError:&error]) {
|
||||
NSLog(@"Failed to start graph: %@", error);
|
||||
}
|
||||
else if (![self.mediapipeGraph waitUntilIdleWithError:&error]) {
|
||||
NSLog(@"Failed to complete graph initial run: %@", error);
|
||||
}
|
||||
|
||||
dispatch_async(_videoQueue, ^{
|
||||
[_cameraSource start];
|
||||
@@ -500,8 +503,9 @@ in our app:
|
||||
}];
|
||||
```
|
||||
|
||||
Note: It is important to start the graph before starting the camera, so that the
|
||||
graph is ready to process frames as soon as the camera starts sending them.
|
||||
Note: It is important to start the graph before starting the camera and wait
|
||||
until completion, so that the graph is ready to process frames as soon as the
|
||||
camera starts sending them.
|
||||
|
||||
Earlier, when we received frames from the camera in the `processVideoFrame`
|
||||
function, we displayed them in the `_liveView` using the `_renderer`. Now, we
|
||||
|
||||
@@ -19,9 +19,10 @@ MediaPipe currently offers the following solutions:
|
||||
Solution | NPM Package | Example
|
||||
----------------- | ----------------------------- | -------
|
||||
[Face Mesh][F-pg] | [@mediapipe/face_mesh][F-npm] | [mediapipe.dev/demo/face_mesh][F-demo]
|
||||
[Face Detection][Fd-pg] | [@mediapipe/face_detection][Fd-npm] | [mediapipe.dev/demo/face_detection][Fd-demo]
|
||||
[Hands][H-pg] | [@mediapipe/hands][H-npm] | [mediapipe.dev/demo/hands][H-demo]
|
||||
[Pose][P-pg] | [@mediapipe/pose][P-npm] | [mediapipe.dev/demo/pose][P-demo]
|
||||
[Holistic][Ho-pg] | [@mediapipe/holistic][Ho-npm] | [mediapipe.dev/demo/holistic][Ho-demo]
|
||||
[Pose][P-pg] | [@mediapipe/pose][P-npm] | [mediapipe.dev/demo/pose][P-demo]
|
||||
|
||||
Click on a solution link above for more information, including API and code
|
||||
snippets.
|
||||
@@ -63,10 +64,12 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
|
||||
|
||||
[Ho-pg]: ../solutions/holistic#javascript-solution-api
|
||||
[F-pg]: ../solutions/face_mesh#javascript-solution-api
|
||||
[Fd-pg]: ../solutions/face_detection#javascript-solution-api
|
||||
[H-pg]: ../solutions/hands#javascript-solution-api
|
||||
[P-pg]: ../solutions/pose#javascript-solution-api
|
||||
[Ho-npm]: https://www.npmjs.com/package/@mediapipe/holistic
|
||||
[F-npm]: https://www.npmjs.com/package/@mediapipe/face_mesh
|
||||
[Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection
|
||||
[H-npm]: https://www.npmjs.com/package/@mediapipe/hands
|
||||
[P-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[draw-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
@@ -74,14 +77,17 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
|
||||
[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/pose
|
||||
[Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic
|
||||
[F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh
|
||||
[Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection
|
||||
[H-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/hands
|
||||
[P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/pose
|
||||
[Ho-pen]: https://code.mediapipe.dev/codepen/holistic
|
||||
[F-pen]: https://code.mediapipe.dev/codepen/face_mesh
|
||||
[Fd-pen]: https://code.mediapipe.dev/codepen/face_detection
|
||||
[H-pen]: https://code.mediapipe.dev/codepen/hands
|
||||
[P-pen]: https://code.mediapipe.dev/codepen/pose
|
||||
[Ho-demo]: https://mediapipe.dev/demo/holistic
|
||||
[F-demo]: https://mediapipe.dev/demo/face_mesh
|
||||
[Fd-demo]: https://mediapipe.dev/demo/face_detection
|
||||
[H-demo]: https://mediapipe.dev/demo/hands
|
||||
[P-demo]: https://mediapipe.dev/demo/pose
|
||||
[npm]: https://www.npmjs.com/package/@mediapipe
|
||||
|
||||
@@ -45,17 +45,23 @@ Tip: Use command `deactivate` to later exit the Python virtual environment.
|
||||
To learn more about configuration options and usage examples, please find
|
||||
details in each solution via the links below:
|
||||
|
||||
* [MediaPipe Face Detection](../solutions/face_detection#python-solution-api)
|
||||
* [MediaPipe Face Mesh](../solutions/face_mesh#python-solution-api)
|
||||
* [MediaPipe Hands](../solutions/hands#python-solution-api)
|
||||
* [MediaPipe Pose](../solutions/pose#python-solution-api)
|
||||
* [MediaPipe Holistic](../solutions/holistic#python-solution-api)
|
||||
* [MediaPipe Objectron](../solutions/objectron#python-solution-api)
|
||||
* [MediaPipe Pose](../solutions/pose#python-solution-api)
|
||||
|
||||
## MediaPipe on Google Colab
|
||||
|
||||
* [MediaPipe Face Detection Colab](https://mediapipe.page.link/face_detection_py_colab)
|
||||
* [MediaPipe Face Mesh Colab](https://mediapipe.page.link/face_mesh_py_colab)
|
||||
* [MediaPipe Hands Colab](https://mediapipe.page.link/hands_py_colab)
|
||||
* [MediaPipe Pose Colab](https://mediapipe.page.link/pose_py_colab)
|
||||
* [MediaPipe Holistic Colab](https://mediapipe.page.link/holistic_py_colab)
|
||||
* [MediaPipe Objectron Colab](https://mediapipe.page.link/objectron_py_colab)
|
||||
* [MediaPipe Pose Colab](https://mediapipe.page.link/pose_py_colab)
|
||||
* [MediaPipe Pose Classification Colab (Basic)](https://mediapipe.page.link/pose_classification_basic)
|
||||
* [MediaPipe Pose Classification Colab (Extended)](https://mediapipe.page.link/pose_classification_extended)
|
||||
|
||||
## MediaPipe Python Framework
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 7.7 KiB |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 11 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 60 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1006 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 11 KiB |
+2
-2
@@ -34,7 +34,7 @@ Hair Segmentation
|
||||
|
||||
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
||||
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
@@ -44,7 +44,7 @@ Hair Segmentation
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | ✅ | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
|
||||
@@ -39,6 +39,169 @@ section.
|
||||
|
||||
|
||||
|
||||
## Solution APIs
|
||||
|
||||
### Configuration Options
|
||||
|
||||
Naming style and availability may differ slightly across platforms/languages.
|
||||
|
||||
#### min_detection_confidence
|
||||
|
||||
Minimum confidence value (`[0.0, 1.0]`) from the face detection model for the
|
||||
detection to be considered successful. Default to `0.5`.
|
||||
|
||||
### Output
|
||||
|
||||
Naming style may differ slightly across platforms/languages.
|
||||
|
||||
#### detections
|
||||
|
||||
Collection of detected faces, where each face is represented as a detection
|
||||
proto message that contains a bounding box and 6 key points (right eye, left
|
||||
eye, nose tip, mouth center, right ear tragion, and left ear tragion). The
|
||||
bounding box is composed of `xmin` and `width` (both normalized to `[0.0, 1.0]`
|
||||
by the image width) and `ymin` and `height` (both normalized to `[0.0, 1.0]` by
|
||||
the image height). Each key point is composed of `x` and `y`, which are
|
||||
normalized to `[0.0, 1.0]` by the image width and height respectively.
|
||||
|
||||
### Python Solution API
|
||||
|
||||
Please first follow general [instructions](../getting_started/python.md) to
|
||||
install MediaPipe Python package, then learn more in the companion
|
||||
[Python Colab](#resources) and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
* [min_detection_confidence](#min_detection_confidence)
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_face_detction = mp.solutions.face_detection
|
||||
|
||||
# For static images:
|
||||
with mp_face_detection.FaceDetection(
|
||||
min_detection_confidence=0.5) as face_detection:
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
# Convert the BGR image to RGB and process it with MediaPipe Face Detection.
|
||||
results = face_detection.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Draw face detections of each face.
|
||||
if not results.detections:
|
||||
continue
|
||||
annotated_image = image.copy()
|
||||
for detection in results.detections:
|
||||
print('Nose tip:')
|
||||
print(mp_face_detection.get_key_point(
|
||||
detection, mp_face_detection.FaceKeyPoint.NOSE_TIP))
|
||||
mp_drawing.draw_detection(annotated_image, detection)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
|
||||
# For webcam input:
|
||||
cap = cv2.VideoCapture(0)
|
||||
with mp_face_detection.FaceDetection(
|
||||
min_detection_confidence=0.5) as face_detection:
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = face_detection.process(image)
|
||||
|
||||
# Draw the face detection annotations on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.detections:
|
||||
for detection in results.detections:
|
||||
mp_drawing.draw_detection(image, detection)
|
||||
cv2.imshow('MediaPipe Face Detection', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
cap.release()
|
||||
```
|
||||
|
||||
### JavaScript Solution API
|
||||
|
||||
Please first see general [introduction](../getting_started/javascript.md) on
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
|
||||
and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
* [minDetectionConfidence](#min_detection_confidence)
|
||||
|
||||
```html
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/face_detection/face_detection.js" crossorigin="anonymous"></script>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="container">
|
||||
<video class="input_video"></video>
|
||||
<canvas class="output_canvas" width="1280px" height="720px"></canvas>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
```
|
||||
|
||||
```javascript
|
||||
<script type="module">
|
||||
const videoElement = document.getElementsByClassName('input_video')[0];
|
||||
const canvasElement = document.getElementsByClassName('output_canvas')[0];
|
||||
const canvasCtx = canvasElement.getContext('2d');
|
||||
|
||||
function onResults(results) {
|
||||
// Draw the overlays.
|
||||
canvasCtx.save();
|
||||
canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
|
||||
canvasCtx.drawImage(
|
||||
results.image, 0, 0, canvasElement.width, canvasElement.height);
|
||||
if (results.detections.length > 0) {
|
||||
drawingUtils.drawRectangle(
|
||||
canvasCtx, results.detections[0].boundingBox,
|
||||
{color: 'blue', lineWidth: 4, fillColor: '#00000000'});
|
||||
drawingUtils.drawLandmarks(canvasCtx, results.detections[0].landmarks, {
|
||||
color: 'red',
|
||||
radius: 5,
|
||||
});
|
||||
}
|
||||
canvasCtx.restore();
|
||||
}
|
||||
|
||||
const faceDetection = new Objectron({locateFile: (file) => {
|
||||
return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${file}`;
|
||||
}});
|
||||
faceDetection.setOptions({
|
||||
minDetectionConfidence: 0.5
|
||||
});
|
||||
faceDetection.onResults(onResults);
|
||||
|
||||
const camera = new Camera(videoElement, {
|
||||
onFrame: async () => {
|
||||
await faceDetection.send({image: videoElement});
|
||||
},
|
||||
width: 1280,
|
||||
height: 720
|
||||
});
|
||||
camera.start();
|
||||
</script>
|
||||
```
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
@@ -108,3 +271,5 @@ to cross-compile and run MediaPipe examples on the
|
||||
([presentation](https://docs.google.com/presentation/d/1YCtASfnYyZtH-41QvnW5iZxELFnf0MF-pPWSLGj8yjQ/present?slide=id.g5bc8aeffdd_1_0))
|
||||
([poster](https://drive.google.com/file/d/1u6aB6wxDY7X2TmeUUKgFydulNtXkb3pu/view))
|
||||
* [Models and model cards](./models.md#face_detection)
|
||||
* [Web demo](https://code.mediapipe.dev/codepen/face_detection)
|
||||
* [Python Colab](https://mediapipe.page.link/face_detection_py_colab)
|
||||
|
||||
+58
-61
@@ -185,8 +185,8 @@ following steps are executed in the given order:
|
||||
The geometry pipeline is implemented as a MediaPipe
|
||||
[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc).
|
||||
For your convenience, the face geometry pipeline calculator is bundled together
|
||||
with the face landmark module into a unified MediaPipe
|
||||
[subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_front_gpu.pbtxt).
|
||||
with corresponding metadata into a unified MediaPipe
|
||||
[subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_from_landmarks.pbtxt).
|
||||
The face geometry format is defined as a Protocol Buffer
|
||||
[message](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto).
|
||||
|
||||
@@ -264,8 +264,8 @@ magnitude of `z` uses roughly the same scale as `x`.
|
||||
### Python Solution API
|
||||
|
||||
Please first follow general [instructions](../getting_started/python.md) to
|
||||
install MediaPipe Python package, then learn more in the companion [Colab] and
|
||||
the following usage example.
|
||||
install MediaPipe Python package, then learn more in the companion
|
||||
[Python Colab](#resources) and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
@@ -281,74 +281,73 @@ mp_drawing = mp.solutions.drawing_utils
|
||||
mp_face_mesh = mp.solutions.face_mesh
|
||||
|
||||
# For static images:
|
||||
face_mesh = mp_face_mesh.FaceMesh(
|
||||
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
||||
with mp_face_mesh.FaceMesh(
|
||||
static_image_mode=True,
|
||||
max_num_faces=1,
|
||||
min_detection_confidence=0.5)
|
||||
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
min_detection_confidence=0.5) as face_mesh:
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Print and draw face mesh landmarks on the image.
|
||||
if not results.multi_face_landmarks:
|
||||
continue
|
||||
annotated_image = image.copy()
|
||||
for face_landmarks in results.multi_face_landmarks:
|
||||
print('face_landmarks:', face_landmarks)
|
||||
mp_drawing.draw_landmarks(
|
||||
image=annotated_image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACE_CONNECTIONS,
|
||||
landmark_drawing_spec=drawing_spec,
|
||||
connection_drawing_spec=drawing_spec)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
face_mesh.close()
|
||||
|
||||
# For webcam input:
|
||||
face_mesh = mp_face_mesh.FaceMesh(
|
||||
min_detection_confidence=0.5, min_tracking_confidence=0.5)
|
||||
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = face_mesh.process(image)
|
||||
|
||||
# Draw the face mesh annotations on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.multi_face_landmarks:
|
||||
# Print and draw face mesh landmarks on the image.
|
||||
if not results.multi_face_landmarks:
|
||||
continue
|
||||
annotated_image = image.copy()
|
||||
for face_landmarks in results.multi_face_landmarks:
|
||||
print('face_landmarks:', face_landmarks)
|
||||
mp_drawing.draw_landmarks(
|
||||
image=image,
|
||||
image=annotated_image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACE_CONNECTIONS,
|
||||
landmark_drawing_spec=drawing_spec,
|
||||
connection_drawing_spec=drawing_spec)
|
||||
cv2.imshow('MediaPipe FaceMesh', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
face_mesh.close()
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
|
||||
# For webcam input:
|
||||
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
||||
cap = cv2.VideoCapture(0)
|
||||
with mp_face_mesh.FaceMesh(
|
||||
min_detection_confidence=0.5,
|
||||
min_tracking_confidence=0.5) as face_mesh:
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = face_mesh.process(image)
|
||||
|
||||
# Draw the face mesh annotations on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.multi_face_landmarks:
|
||||
for face_landmarks in results.multi_face_landmarks:
|
||||
mp_drawing.draw_landmarks(
|
||||
image=image,
|
||||
landmark_list=face_landmarks,
|
||||
connections=mp_face_mesh.FACE_CONNECTIONS,
|
||||
landmark_drawing_spec=drawing_spec,
|
||||
connection_drawing_spec=drawing_spec)
|
||||
cv2.imshow('MediaPipe FaceMesh', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
cap.release()
|
||||
```
|
||||
|
||||
### JavaScript Solution API
|
||||
|
||||
Please first see general [introduction](../getting_started/javascript.md) on
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo] and the
|
||||
following usage example.
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
|
||||
and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
@@ -503,7 +502,5 @@ only works for a single face. For visual reference, please refer to *Fig. 4*.
|
||||
[OBJ](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.obj),
|
||||
[UV visualization](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png)
|
||||
* [Models and model cards](./models.md#face_mesh)
|
||||
|
||||
[Colab]:https://mediapipe.page.link/face_mesh_py_colab
|
||||
|
||||
[web demo]:https://code.mediapipe.dev/codepen/face_mesh
|
||||
* [Web demo](https://code.mediapipe.dev/codepen/face_mesh)
|
||||
* [Python Colab](https://mediapipe.page.link/face_mesh_py_colab)
|
||||
|
||||
+70
-74
@@ -91,13 +91,14 @@ To detect initial hand locations, we designed a
|
||||
mobile real-time uses in a manner similar to the face detection model in
|
||||
[MediaPipe Face Mesh](./face_mesh.md). Detecting hands is a decidedly complex
|
||||
task: our
|
||||
[model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite) has
|
||||
to work across a variety of hand sizes with a large scale span (~20x) relative
|
||||
to the image frame and be able to detect occluded and self-occluded hands.
|
||||
Whereas faces have high contrast patterns, e.g., in the eye and mouth region,
|
||||
the lack of such features in hands makes it comparatively difficult to detect
|
||||
them reliably from their visual features alone. Instead, providing additional
|
||||
context, like arm, body, or person features, aids accurate hand localization.
|
||||
[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection.tflite)
|
||||
has to work across a variety of hand sizes with a large scale span (~20x)
|
||||
relative to the image frame and be able to detect occluded and self-occluded
|
||||
hands. Whereas faces have high contrast patterns, e.g., in the eye and mouth
|
||||
region, the lack of such features in hands makes it comparatively difficult to
|
||||
detect them reliably from their visual features alone. Instead, providing
|
||||
additional context, like arm, body, or person features, aids accurate hand
|
||||
localization.
|
||||
|
||||
Our method addresses the above challenges using different strategies. First, we
|
||||
train a palm detector instead of a hand detector, since estimating bounding
|
||||
@@ -119,7 +120,7 @@ just 86.22%.
|
||||
### Hand Landmark Model
|
||||
|
||||
After the palm detection over the whole image our subsequent hand landmark
|
||||
[model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite)
|
||||
[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite)
|
||||
performs precise keypoint localization of 21 3D hand-knuckle coordinates inside
|
||||
the detected hand regions via regression, that is direct coordinate prediction.
|
||||
The model learns a consistent internal hand pose representation and is robust
|
||||
@@ -136,11 +137,9 @@ to the corresponding 3D coordinates.
|
||||
:--------------------------------------------------------: |
|
||||
*Fig 2. 21 hand landmarks.* |
|
||||
|
||||
|  |
|
||||
| :-------------------------------------------------------------------------: |
|
||||
| *Fig 3. Top: Aligned hand crops passed to the tracking network with ground |
|
||||
: truth annotation. Bottom\: Rendered synthetic hand images with ground truth :
|
||||
: annotation.* :
|
||||
 |
|
||||
:-------------------------------------------------------------------------: |
|
||||
*Fig 3. Top: Aligned hand crops passed to the tracking network with ground truth annotation. Bottom: Rendered synthetic hand images with ground truth annotation.* |
|
||||
|
||||
## Solution APIs
|
||||
|
||||
@@ -206,8 +205,8 @@ is not the case, please swap the handedness output in the application.
|
||||
### Python Solution API
|
||||
|
||||
Please first follow general [instructions](../getting_started/python.md) to
|
||||
install MediaPipe Python package, then learn more in the companion [Colab] and
|
||||
the following usage example.
|
||||
install MediaPipe Python package, then learn more in the companion
|
||||
[Python Colab](#resources) and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
@@ -223,74 +222,73 @@ mp_drawing = mp.solutions.drawing_utils
|
||||
mp_hands = mp.solutions.hands
|
||||
|
||||
# For static images:
|
||||
hands = mp_hands.Hands(
|
||||
with mp_hands.Hands(
|
||||
static_image_mode=True,
|
||||
max_num_hands=2,
|
||||
min_detection_confidence=0.5)
|
||||
for idx, file in enumerate(file_list):
|
||||
# Read an image, flip it around y-axis for correct handedness output (see
|
||||
# above).
|
||||
image = cv2.flip(cv2.imread(file), 1)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
min_detection_confidence=0.5) as hands:
|
||||
for idx, file in enumerate(file_list):
|
||||
# Read an image, flip it around y-axis for correct handedness output (see
|
||||
# above).
|
||||
image = cv2.flip(cv2.imread(file), 1)
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Print handedness and draw hand landmarks on the image.
|
||||
print('Handedness:', results.multi_handedness)
|
||||
if not results.multi_hand_landmarks:
|
||||
continue
|
||||
image_hight, image_width, _ = image.shape
|
||||
annotated_image = image.copy()
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
print('hand_landmarks:', hand_landmarks)
|
||||
print(
|
||||
f'Index finger tip coordinates: (',
|
||||
f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].x * image_width}, '
|
||||
f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * image_hight})'
|
||||
)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
cv2.imwrite(
|
||||
'/tmp/annotated_image' + str(idx) + '.png', cv2.flip(annotated_image, 1))
|
||||
hands.close()
|
||||
# Print handedness and draw hand landmarks on the image.
|
||||
print('Handedness:', results.multi_handedness)
|
||||
if not results.multi_hand_landmarks:
|
||||
continue
|
||||
image_height, image_width, _ = image.shape
|
||||
annotated_image = image.copy()
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
print('hand_landmarks:', hand_landmarks)
|
||||
print(
|
||||
f'Index finger tip coordinates: (',
|
||||
f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].x * image_width}, '
|
||||
f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * image_height})'
|
||||
)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
cv2.imwrite(
|
||||
'/tmp/annotated_image' + str(idx) + '.png', cv2.flip(annotated_image, 1))
|
||||
|
||||
# For webcam input:
|
||||
hands = mp_hands.Hands(
|
||||
min_detection_confidence=0.5, min_tracking_confidence=0.5)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
with mp_hands.Hands(
|
||||
min_detection_confidence=0.5,
|
||||
min_tracking_confidence=0.5) as hands:
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = hands.process(image)
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = hands.process(image)
|
||||
|
||||
# Draw the hand annotations on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.multi_hand_landmarks:
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
mp_drawing.draw_landmarks(
|
||||
image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Hands', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
hands.close()
|
||||
# Draw the hand annotations on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.multi_hand_landmarks:
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
mp_drawing.draw_landmarks(
|
||||
image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Hands', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
cap.release()
|
||||
```
|
||||
|
||||
### JavaScript Solution API
|
||||
|
||||
Please first see general [introduction](../getting_started/javascript.md) on
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo] and a
|
||||
[fun application], and the following usage example.
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
|
||||
and a [fun application], and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
@@ -425,8 +423,6 @@ it, in the graph file modify the option of `ConstantSidePacketCalculator`.
|
||||
[MediaPipe Hands: On-device Real-time Hand Tracking](https://arxiv.org/abs/2006.10214)
|
||||
([presentation](https://www.youtube.com/watch?v=I-UOrvxxXEk))
|
||||
* [Models and model cards](./models.md#hands)
|
||||
|
||||
[Colab]:https://mediapipe.page.link/hands_py_colab
|
||||
|
||||
[web demo]:https://code.mediapipe.dev/codepen/hands
|
||||
[fun application]:https://code.mediapipe.dev/codepen/defrost
|
||||
* [Web demo](https://code.mediapipe.dev/codepen/hands)
|
||||
* [Fun application](https://code.mediapipe.dev/codepen/defrost)
|
||||
* [Python Colab](https://mediapipe.page.link/hands_py_colab)
|
||||
|
||||
+61
-62
@@ -201,8 +201,8 @@ A list of 21 hand landmarks on the right hand, in the same representation as
|
||||
### Python Solution API
|
||||
|
||||
Please first follow general [instructions](../getting_started/python.md) to
|
||||
install MediaPipe Python package, then learn more in the companion [Colab] and
|
||||
the following usage example.
|
||||
install MediaPipe Python package, then learn more in the companion
|
||||
[Python Colab](#resources) and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
@@ -219,74 +219,75 @@ mp_drawing = mp.solutions.drawing_utils
|
||||
mp_holistic = mp.solutions.holistic
|
||||
|
||||
# For static images:
|
||||
holistic = mp_holistic.Holistic(static_image_mode=True)
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
image_hight, image_width, _ = image.shape
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
with mp_holistic.Holistic(static_image_mode=True) as holistic:
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
image_height, image_width, _ = image.shape
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
if results.pose_landmarks:
|
||||
print(
|
||||
f'Nose coordinates: ('
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_hight})'
|
||||
)
|
||||
# Draw pose, left and right hands, and face landmarks on the image.
|
||||
annotated_image = image.copy()
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
holistic.close()
|
||||
if results.pose_landmarks:
|
||||
print(
|
||||
f'Nose coordinates: ('
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
|
||||
)
|
||||
# Draw pose, left and right hands, and face landmarks on the image.
|
||||
annotated_image = image.copy()
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
# Use mp_holistic.UPPER_BODY_POSE_CONNECTIONS for drawing below when
|
||||
# upper_body_only is set to True.
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
|
||||
# For webcam input:
|
||||
holistic = mp_holistic.Holistic(
|
||||
min_detection_confidence=0.5, min_tracking_confidence=0.5)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
with mp_holistic.Holistic(
|
||||
min_detection_confidence=0.5,
|
||||
min_tracking_confidence=0.5) as holistic:
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = holistic.process(image)
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = holistic.process(image)
|
||||
|
||||
# Draw landmark annotation on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Holistic', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
holistic.close()
|
||||
# Draw landmark annotation on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Holistic', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
cap.release()
|
||||
```
|
||||
|
||||
### JavaScript Solution API
|
||||
|
||||
Please first see general [introduction](../getting_started/javascript.md) on
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo] and the
|
||||
following usage example.
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
|
||||
and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
@@ -407,7 +408,5 @@ on how to build MediaPipe examples.
|
||||
* Google AI Blog:
|
||||
[MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
|
||||
* [Models and model cards](./models.md#holistic)
|
||||
|
||||
[Colab]:https://mediapipe.page.link/holistic_py_colab
|
||||
|
||||
[web demo]:https://code.mediapipe.dev/codepen/holistic
|
||||
* [Web demo](https://code.mediapipe.dev/codepen/holistic)
|
||||
* [Python Colab](https://mediapipe.page.link/holistic_py_colab)
|
||||
|
||||
@@ -117,6 +117,25 @@ Please first see general instructions for
|
||||
* Android target (or download prebuilt [ARM64 APK](https://drive.google.com/file/d/1KnaBBoKpCHR73nOBJ4fL_YdWVTAcwe6L/view?usp=sharing)):
|
||||
[`mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking:instantmotiontracking`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD)
|
||||
|
||||
* Assets rendered by the [GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc) must be preprocessed into an OpenGL-ready custom .uuu format. This can be done
|
||||
for user assets as follows:
|
||||
> First run
|
||||
>
|
||||
> ```shell
|
||||
> ./mediapipe/graphs/object_detection_3d/obj_parser/obj_cleanup.sh [INPUT_DIR] [INTERMEDIATE_OUTPUT_DIR]
|
||||
> ```
|
||||
> and then run
|
||||
>
|
||||
> ```build
|
||||
> bazel run -c opt mediapipe/graphs/object_detection_3d/obj_parser:ObjParser -- input_dir=[INTERMEDIATE_OUTPUT_DIR] output_dir=[OUTPUT_DIR]
|
||||
> ```
|
||||
> INPUT_DIR should be the folder with initial asset .obj files to be processed,
|
||||
> and OUTPUT_DIR is the folder where the processed asset .uuu file will be placed.
|
||||
>
|
||||
> Note: ObjParser combines all .obj files found in the given directory into a
|
||||
> single .uuu animation file, using the order given by sorting the filenames alphanumerically. Also the ObjParser directory inputs must be given as
|
||||
> absolute paths, not relative paths. See parser utility library at [`mediapipe/graphs/object_detection_3d/obj_parser/`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/obj_parser/) for more details.
|
||||
|
||||
## Resources
|
||||
|
||||
* Google Developers Blog:
|
||||
|
||||
@@ -41,8 +41,9 @@ nav_order: 30
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
|
||||
* Hand landmark model:
|
||||
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite),
|
||||
[TFLite model (sparse)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_sparse.tflite),
|
||||
[TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
|
||||
* [Model card](https://mediapipe.page.link/handmc)
|
||||
* [Model card](https://mediapipe.page.link/handmc), [Model card (sparse)](https://mediapipe.page.link/handmc-sparse)
|
||||
|
||||
### [Pose](https://google.github.io/mediapipe/solutions/pose)
|
||||
|
||||
@@ -73,12 +74,12 @@ nav_order: 30
|
||||
|
||||
### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
|
||||
|
||||
* [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
|
||||
* [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
|
||||
* [TFLite model for cameras](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_camera.tflite)
|
||||
* [TFLite model for cups](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_cup.tflite)
|
||||
* [Single-stage TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers_1stage.tflite)
|
||||
* [Single-stage TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair_1stage.tflite)
|
||||
* [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_sneakers.tflite)
|
||||
* [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_chair.tflite)
|
||||
* [TFLite model for cameras](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_camera.tflite)
|
||||
* [TFLite model for cups](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_cup.tflite)
|
||||
* [Single-stage TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite)
|
||||
* [Single-stage TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite)
|
||||
* [Model card](https://mediapipe.page.link/objectron-mc)
|
||||
|
||||
### [KNIFT](https://google.github.io/mediapipe/solutions/knift)
|
||||
|
||||
@@ -186,6 +186,175 @@ trained our 3D object detection models. The technical details of the Objectron
|
||||
dataset, including usage and tutorials, are available on
|
||||
the [dataset website](https://github.com/google-research-datasets/Objectron/).
|
||||
|
||||
## Solution APIs
|
||||
|
||||
### Cross-platform Configuration Options
|
||||
|
||||
Naming style and availability may differ slightly across platforms/languages.
|
||||
|
||||
#### static_image_mode
|
||||
|
||||
If set to `false`, the solution treats the input images as a video stream. It
|
||||
will try to detect objects in the very first images, and upon successful
|
||||
detection further localizes the 3D bounding box landmarks. In subsequent images,
|
||||
once all [max_num_objects](#max_num_objects) objects are detected and the
|
||||
corresponding 3D bounding box landmarks are localized, it simply tracks those
|
||||
landmarks without invoking another detection until it loses track of any of the
|
||||
objects. This reduces latency and is ideal for processing video frames. If set
|
||||
to `true`, object detection runs every input image, ideal for processing a batch
|
||||
of static, possibly unrelated, images. Default to `false`.
|
||||
|
||||
#### max_num_objects
|
||||
|
||||
Maximum number of objects to detect. Default to `5`.
|
||||
|
||||
#### min_detection_confidence
|
||||
|
||||
Minimum confidence value (`[0.0, 1.0]`) from the object-detection model for the
|
||||
detection to be considered successful. Default to `0.5`.
|
||||
|
||||
#### min_tracking_confidence
|
||||
|
||||
Minimum confidence value (`[0.0, 1.0]`) from the landmark-tracking model for the
|
||||
3D bounding box landmarks to be considered tracked successfully, or otherwise
|
||||
object detection will be invoked automatically on the next input image. Setting
|
||||
it to a higher value can increase robustness of the solution, at the expense of
|
||||
a higher latency. Ignored if [static_image_mode](#static_image_mode) is `true`,
|
||||
where object detection simply runs on every image. Default to `0.99`.
|
||||
|
||||
#### model_name
|
||||
|
||||
Name of the model to use for predicting 3D bounding box landmarks. Currently supports
|
||||
`{'Shoe', 'Chair', 'Cup', 'Camera'}`.
|
||||
|
||||
#### focal_length
|
||||
|
||||
Camera focal length `(fx, fy)`, by default is defined in
|
||||
[NDC space](#ndc-space). To use focal length `(fx_pixel, fy_pixel)` in
|
||||
[pixel space](#pixel-space), users should provide `image_size` = `(image_width,
|
||||
image_height)` to enable conversions inside the API. For further details about
|
||||
NDC and pixel space, please see [Coordinate Systems](#coordinate-systems).
|
||||
|
||||
#### principal_point
|
||||
|
||||
Camera principal point `(px, py)`, by default is defined in
|
||||
[NDC space](#ndc-space). To use principal point `(px_pixel, py_pixel)` in
|
||||
[pixel space](#pixel-space), users should provide `image_size` = `(image_width,
|
||||
image_height)` to enable conversions inside the API. For further details about
|
||||
NDC and pixel space, please see [Coordinate Systems](#coordinate-systems).
|
||||
|
||||
#### image_size
|
||||
|
||||
(**Optional**) size `(image_width, image_height)` of the input image, **ONLY**
|
||||
needed when use `focal_length` and `principal_point` in pixel space.
|
||||
|
||||
### Output
|
||||
|
||||
<!-- Naming style may differ slightly across platforms/languages. -->
|
||||
|
||||
#### detected_objects
|
||||
|
||||
A list of detected 3D bounding box. Each 3D bounding box consists of the
|
||||
following:
|
||||
|
||||
* `landmarks_2d` : 2D landmarks of the object's 3D bounding box. The landmark
|
||||
coordinates are normalized to `[0.0, 1.0]` by the image width and height
|
||||
respectively.
|
||||
|
||||
* `landmarks_3d` : 3D landmarks of the object's 3D bounding box. The landmark
|
||||
coordinates are represented in [camera coordinate](#camera-coordinate)
|
||||
frame.
|
||||
|
||||
* `rotation` : rotation matrix from object coordinate frame to camera
|
||||
coordinate frame.
|
||||
|
||||
* `translation` : translation vector from object coordinate frame to camera
|
||||
coordinate frame.
|
||||
|
||||
* `scale` : relative scale of the object along `x`, `y` and `z` directions.
|
||||
|
||||
## Python Solution API
|
||||
|
||||
Please first follow general [instructions](../getting_started/python.md) to
|
||||
install MediaPipe Python package, then learn more in the companion
|
||||
[Python Colab](#resources) and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
* [static_image_mode](#static_image_mode)
|
||||
* [max_num_objects](#max_num_objects)
|
||||
* [min_detection_confidence](#min_detection_confidence)
|
||||
* [min_tracking_confidence](#min_tracking_confidence)
|
||||
* [model_name](#model_name)
|
||||
* [focal_length](#focal_length)
|
||||
* [principal_point](#principal_point)
|
||||
* [image_size](#image_size)
|
||||
|
||||
```python
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_objectron = mp.solutions.objectron
|
||||
|
||||
# For static images:
|
||||
with mp_objectron.Objectron(static_image_mode=True,
|
||||
max_num_objects=5,
|
||||
min_detection_confidence=0.5,
|
||||
model_name='Shoe') as objectron:
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
# Convert the BGR image to RGB and process it with MediaPipe Objectron.
|
||||
results = objectron.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Draw box landmarks.
|
||||
if not results.detected_objects:
|
||||
print(f'No box landmarks detected on {file}')
|
||||
continue
|
||||
print(f'Box landmarks of {file}:')
|
||||
annotated_image = image.copy()
|
||||
for detected_object in results.detected_objects:
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, detected_object.landmarks_2d, mp_objectron.BOX_CONNECTIONS)
|
||||
mp_drawing.draw_axis(annotated_image, detected_object.rotation,
|
||||
detected_object.translation)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
|
||||
# For webcam input:
|
||||
cap = cv2.VideoCapture(0)
|
||||
with mp_objectron.Objectron(static_image_mode=False,
|
||||
max_num_objects=5,
|
||||
min_detection_confidence=0.5,
|
||||
min_tracking_confidence=0.99,
|
||||
model_name='Shoe') as objectron:
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
|
||||
# Convert the BGR image to RGB.
|
||||
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = objectron.process(image)
|
||||
|
||||
# Draw the box landmarks on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
if results.detected_objects:
|
||||
for detected_object in results.detected_objects:
|
||||
mp_drawing.draw_landmarks(
|
||||
image, detected_object.landmarks_2d, mp_objectron.BOX_CONNECTIONS)
|
||||
mp_drawing.draw_axis(image, detected_object.rotation,
|
||||
detected_object.translation)
|
||||
cv2.imshow('MediaPipe Objectron', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
cap.release()
|
||||
```
|
||||
|
||||
## Example Apps
|
||||
|
||||
Please first see general instructions for
|
||||
@@ -259,6 +428,104 @@ to visualize its associated subgraphs, please see
|
||||
|
||||
* iOS target: Not available
|
||||
|
||||
### Assets
|
||||
|
||||
Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc) using a parsing of the sequenced .obj file
|
||||
format into a custom .uuu format. This can be done for user assets as follows:
|
||||
> First run
|
||||
>
|
||||
> ```shell
|
||||
> ./mediapipe/graphs/object_detection_3d/obj_parser/obj_cleanup.sh [INPUT_DIR] [INTERMEDIATE_OUTPUT_DIR]
|
||||
> ```
|
||||
> and then run
|
||||
>
|
||||
> ```build
|
||||
> bazel run -c opt mediapipe/graphs/object_detection_3d/obj_parser:ObjParser -- input_dir=[INTERMEDIATE_OUTPUT_DIR] output_dir=[OUTPUT_DIR]
|
||||
> ```
|
||||
> INPUT_DIR should be the folder with initial asset .obj files to be processed,
|
||||
> and OUTPUT_DIR is the folder where the processed asset .uuu file will be placed.
|
||||
>
|
||||
> Note: ObjParser combines all .obj files found in the given directory into a
|
||||
> single .uuu animation file, using the order given by sorting the filenames alphanumerically. Also the ObjParser directory inputs must be given as
|
||||
> absolute paths, not relative paths. See parser utility library at [`mediapipe/graphs/object_detection_3d/obj_parser/`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/obj_parser/) for more details.
|
||||
|
||||
### Coordinate Systems
|
||||
|
||||
#### Object Coordinate
|
||||
|
||||
Each object has its object coordinate frame. We use the below object coordinate
|
||||
definition, with `+x` pointing right, `+y` pointing up and `+z` pointing front,
|
||||
origin is at the center of the 3D bounding box.
|
||||
|
||||
|
||||
|
||||
#### Camera Coordinate
|
||||
|
||||
A 3D object is parameterized by its `scale` and `rotation`, `translation` with
|
||||
regard to the camera coordinate frame. In this API we use the below camera
|
||||
coordinate definition, with `+x` pointing right, `+y` pointing up and `-z`
|
||||
pointing to the scene.
|
||||
|
||||
|
||||
|
||||
To work with box landmarks, one can first derive landmark coordinates in object
|
||||
frame by scaling a origin centered unit box with `scale`, then transform to
|
||||
camera frame by applying `rotation` and `translation`:
|
||||
|
||||
```
|
||||
landmarks_3d = rotation * scale * unit_box + translation
|
||||
```
|
||||
|
||||
#### NDC Space
|
||||
|
||||
In this API we use
|
||||
[NDC(normalized device coordinates)](http://www.songho.ca/opengl/gl_projectionmatrix.html)
|
||||
as an intermediate space when projecting points from 3D to 2D. In NDC space,
|
||||
`x`, `y` are confined to `[-1, 1]`.
|
||||
|
||||
|
||||
|
||||
By default the camera parameters `(fx, fy)` and `(px, py)` are defined in NDC
|
||||
space. Given `(X, Y, Z)` of 3D points in camera coordinate, one can project 3D
|
||||
points to NDC space as follows:
|
||||
|
||||
```
|
||||
x_ndc = -fx * X / Z + px
|
||||
y_ndc = -fy * Y / Z + py
|
||||
z_ndc = 1 / Z
|
||||
```
|
||||
|
||||
#### Pixel Space
|
||||
|
||||
In this API we set upper-left coner of an image as the origin of pixel
|
||||
coordinate. One can convert from NDC to pixel space as follows:
|
||||
|
||||
```
|
||||
x_pixel = (1 + x_ndc) / 2.0 * image_width
|
||||
y_pixel = (1 - y_ndc) / 2.0 * image_height
|
||||
```
|
||||
|
||||
Alternatively one can directly project from camera coordinate to pixel
|
||||
coordinate with camera parameters `(fx_pixel, fy_pixel)` and `(px_pixel,
|
||||
py_pixel)` defined in pixel space as follows:
|
||||
|
||||
```
|
||||
x_pixel = -fx_pixel * X / Z + px_pixel
|
||||
y_pixel = fy_pixel * Y / Z + py_pixel
|
||||
```
|
||||
|
||||
Conversion of camera parameters from pixel space to NDC space:
|
||||
|
||||
```
|
||||
fx = fx_pixel * 2.0 / image_width
|
||||
fy = fy_pixel * 2.0 / image_height
|
||||
```
|
||||
|
||||
```
|
||||
px = -px_pixel * 2.0 / image_width + 1.0
|
||||
py = -py_pixel * 2.0 / image_height + 1.0
|
||||
```
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
@@ -271,3 +538,4 @@ to visualize its associated subgraphs, please see
|
||||
[Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
|
||||
([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
|
||||
* [Models and model cards](./models.md#objectron)
|
||||
* [Python Colab](https://mediapipe.page.link/objectron_py_colab)
|
||||
|
||||
+184
-61
@@ -21,13 +21,15 @@ nav_order: 5
|
||||
## Overview
|
||||
|
||||
Human pose estimation from video plays a critical role in various applications
|
||||
such as quantifying physical exercises, sign language recognition, and full-body
|
||||
gesture control. For example, it can form the basis for yoga, dance, and fitness
|
||||
applications. It can also enable the overlay of digital content and information
|
||||
on top of the physical world in augmented reality.
|
||||
such as
|
||||
[quantifying physical exercises](#pose-classification-and-repetition-counting),
|
||||
sign language recognition, and full-body gesture control. For example, it can
|
||||
form the basis for yoga, dance, and fitness applications. It can also enable the
|
||||
overlay of digital content and information on top of the physical world in
|
||||
augmented reality.
|
||||
|
||||
MediaPipe Pose is a ML solution for high-fidelity body pose tracking, inferring
|
||||
33 2D landmarks on the whole body (or 25 upper-body landmarks) from RGB video
|
||||
33 3D landmarks on the whole body (or 25 upper-body landmarks) from RGB video
|
||||
frames utilizing our
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
research that also powers the
|
||||
@@ -35,7 +37,7 @@ research that also powers the
|
||||
Current state-of-the-art approaches rely primarily on powerful desktop
|
||||
environments for inference, whereas our method achieves real-time performance on
|
||||
most modern [mobile phones](#mobile), [desktops/laptops](#desktop), in
|
||||
[python](#python) and even on the [web](#web).
|
||||
[python](#python-solution-api) and even on the [web](#javascript-solution-api).
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------: |
|
||||
@@ -92,7 +94,7 @@ hip midpoints.
|
||||
:----------------------------------------------------------------------------------------------------: |
|
||||
*Fig 2. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
|
||||
|
||||
### Pose Landmark Model (BlazePose Tracker)
|
||||
### Pose Landmark Model (BlazePose GHUM 3D)
|
||||
|
||||
The landmark model in MediaPipe Pose comes in two versions: a full-body model
|
||||
that predicts the location of 33 pose landmarks (see figure below), and an
|
||||
@@ -163,16 +165,21 @@ A list of pose landmarks. Each lanmark consists of the following:
|
||||
|
||||
* `x` and `y`: Landmark coordinates normalized to `[0.0, 1.0]` by the image
|
||||
width and height respectively.
|
||||
* `z`: Should be discarded as currently the model is not fully trained to
|
||||
predict depth, but this is something on the roadmap.
|
||||
* `z`: Represents the landmark depth with the depth at the midpoint of hips
|
||||
being the origin, and the smaller the value the closer the landmark is to
|
||||
the camera. The magnitude of `z` uses roughly the same scale as `x`.
|
||||
|
||||
Note: `z` is predicted only in full-body mode, and should be discarded when
|
||||
[upper_body_only](#upper_body_only) is `true`.
|
||||
|
||||
* `visibility`: A value in `[0.0, 1.0]` indicating the likelihood of the
|
||||
landmark being visible (present and not occluded) in the image.
|
||||
|
||||
### Python Solution API
|
||||
|
||||
Please first follow general [instructions](../getting_started/python.md) to
|
||||
install MediaPipe Python package, then learn more in the companion [Colab] and
|
||||
the following usage example.
|
||||
install MediaPipe Python package, then learn more in the companion
|
||||
[Python Colab](#resources) and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
@@ -189,64 +196,65 @@ mp_drawing = mp.solutions.drawing_utils
|
||||
mp_pose = mp.solutions.pose
|
||||
|
||||
# For static images:
|
||||
pose = mp_pose.Pose(
|
||||
static_image_mode=True, min_detection_confidence=0.5)
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
image_hight, image_width, _ = image.shape
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
with mp_pose.Pose(
|
||||
static_image_mode=True, min_detection_confidence=0.5) as pose:
|
||||
for idx, file in enumerate(file_list):
|
||||
image = cv2.imread(file)
|
||||
image_height, image_width, _ = image.shape
|
||||
# Convert the BGR image to RGB before processing.
|
||||
results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
if not results.pose_landmarks:
|
||||
continue
|
||||
print(
|
||||
f'Nose coordinates: ('
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_hight})'
|
||||
)
|
||||
# Draw pose landmarks on the image.
|
||||
annotated_image = image.copy()
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
pose.close()
|
||||
if not results.pose_landmarks:
|
||||
continue
|
||||
print(
|
||||
f'Nose coordinates: ('
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
|
||||
f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
|
||||
)
|
||||
# Draw pose landmarks on the image.
|
||||
annotated_image = image.copy()
|
||||
# Use mp_pose.UPPER_BODY_POSE_CONNECTIONS for drawing below when
|
||||
# upper_body_only is set to True.
|
||||
mp_drawing.draw_landmarks(
|
||||
annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
|
||||
|
||||
# For webcam input:
|
||||
pose = mp_pose.Pose(
|
||||
min_detection_confidence=0.5, min_tracking_confidence=0.5)
|
||||
cap = cv2.VideoCapture(0)
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
with mp_pose.Pose(
|
||||
min_detection_confidence=0.5,
|
||||
min_tracking_confidence=0.5) as pose:
|
||||
while cap.isOpened():
|
||||
success, image = cap.read()
|
||||
if not success:
|
||||
print("Ignoring empty camera frame.")
|
||||
# If loading a video, use 'break' instead of 'continue'.
|
||||
continue
|
||||
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = pose.process(image)
|
||||
# Flip the image horizontally for a later selfie-view display, and convert
|
||||
# the BGR image to RGB.
|
||||
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
|
||||
# To improve performance, optionally mark the image as not writeable to
|
||||
# pass by reference.
|
||||
image.flags.writeable = False
|
||||
results = pose.process(image)
|
||||
|
||||
# Draw the pose annotation on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Pose', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
pose.close()
|
||||
# Draw the pose annotation on the image.
|
||||
image.flags.writeable = True
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
mp_drawing.draw_landmarks(
|
||||
image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
|
||||
cv2.imshow('MediaPipe Pose', image)
|
||||
if cv2.waitKey(5) & 0xFF == 27:
|
||||
break
|
||||
cap.release()
|
||||
```
|
||||
|
||||
### JavaScript Solution API
|
||||
|
||||
Please first see general [introduction](../getting_started/javascript.md) on
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo] and the
|
||||
following usage example.
|
||||
MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
|
||||
and the following usage example.
|
||||
|
||||
Supported configuration options:
|
||||
|
||||
@@ -379,6 +387,121 @@ on how to build MediaPipe examples.
|
||||
* Target:
|
||||
[`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
|
||||
|
||||
## Pose Classification and Repetition Counting
|
||||
|
||||
One of the applications
|
||||
[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
can enable is fitness. More specifically - pose classification and repetition
|
||||
counting. In this section we'll provide basic guidance on building a custom pose
|
||||
classifier with the help of a
|
||||
[Colab](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
|
||||
and wrap it in a simple
|
||||
[fitness app](https://mediapipe.page.link/mlkit-pose-classification-demo-app)
|
||||
powered by [ML Kit](https://developers.google.com/ml-kit). Push-ups and squats
|
||||
are used for demonstration purposes as the most common exercises.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 4. Pose classification and repetition counting with MediaPipe Pose.* |
|
||||
|
||||
We picked the
|
||||
[k-nearest neighbors algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm)
|
||||
(k-NN) as the classifier. It's simple and easy to start with. The algorithm
|
||||
determines the object's class based on the closest samples in the training set.
|
||||
To build it, one needs to:
|
||||
|
||||
* Collect image samples of the target exercises and run pose prediction on
|
||||
them,
|
||||
* Convert obtained pose landmarks to a representation suitable for the k-NN
|
||||
classifier and form a training set,
|
||||
* Perform the classification itself followed by repetition counting.
|
||||
|
||||
### Training Set
|
||||
|
||||
To build a good classifier appropriate samples should be collected for the
|
||||
training set: about a few hundred samples for each terminal state of each
|
||||
exercise (e.g., "up" and "down" positions for push-ups). It's important that
|
||||
collected samples cover different camera angles, environment conditions, body
|
||||
shapes, and exercise variations.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 5. Two terminal states of push-ups.* |
|
||||
|
||||
To transform samples into a k-NN classifier training set, either
|
||||
[basic](https://drive.google.com/file/d/1z4IM8kG6ipHN6keadjD-F6vMiIIgViKK/view?usp=sharing)
|
||||
or
|
||||
[extended](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
|
||||
Colab could be used. They both use the
|
||||
[Python Solution API](#python-solution-api) to run the BlazePose models on given
|
||||
images and dump predicted pose landmarks to a CSV file. Additionally, the
|
||||
extended Colab provides useful tools to find outliers (e.g., wrongly predicted
|
||||
poses) and underrepresented classes (e.g., not covering all camera angles) by
|
||||
classifying each sample against the entire training set. After that, you'll be
|
||||
able to test the classifier on an arbitrary video right in the Colab.
|
||||
|
||||
### Classification
|
||||
|
||||
Code of the classifier is available both in the
|
||||
[extended](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
|
||||
Colab and in the
|
||||
[ML Kit demo app](https://mediapipe.page.link/mlkit-pose-classification-demo-app).
|
||||
Please refer to them for details of the approach described below.
|
||||
|
||||
The k-NN algorithm used for pose classification requires a feature vector
|
||||
representation of each sample and a metric to compute the distance between two
|
||||
such vectors to find the nearest pose samples to a target one.
|
||||
|
||||
To convert pose landmarks to a feature vector, we use pairwise distances between
|
||||
predefined lists of pose joints, such as distances between wrist and shoulder,
|
||||
ankle and hip, and two wrists. Since the algorithm relies on distances, all
|
||||
poses are normalized to have the same torso size and vertical torso orientation
|
||||
before the conversion.
|
||||
|
||||
 |
|
||||
:--------------------------------------------------------------------------------------------------------: |
|
||||
*Fig 6. Main pairwise distances used for the pose feature vector.* |
|
||||
|
||||
To get a better classification result, k-NN search is invoked twice with
|
||||
different distance metrics:
|
||||
|
||||
* First, to filter out samples that are almost the same as the target one but
|
||||
have only a few different values in the feature vector (which means
|
||||
differently bent joints and thus other pose class), minimum per-coordinate
|
||||
distance is used as distance metric,
|
||||
* Then average per-coordinate distance is used to find the nearest pose
|
||||
cluster among those from the first search.
|
||||
|
||||
Finally, we apply
|
||||
[exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average)
|
||||
(EMA) smoothing to level any noise from pose prediction or classification. To do
|
||||
that, we search not only for the nearest pose cluster, but we calculate a
|
||||
probability for each of them and use it for smoothing over time.
|
||||
|
||||
### Repetition Counter
|
||||
|
||||
To count the repetitions, the algorithm monitors the probability of a target
|
||||
pose class. Let's take push-ups with its "up" and "down" terminal states:
|
||||
|
||||
* When the probability of the "down" pose class passes a certain threshold for
|
||||
the first time, the algorithm marks that the "down" pose class is entered.
|
||||
* Once the probability drops below the threshold, the algorithm marks that the
|
||||
"down" pose class has been exited and increases the counter.
|
||||
|
||||
To avoid cases when the probability fluctuates around the threshold (e.g., when
|
||||
the user pauses between "up" and "down" states) causing phantom counts, the
|
||||
threshold used to detect when the state is exited is actually slightly lower
|
||||
than the one used to detect when the state is entered. It creates an interval
|
||||
where the pose class and the counter can't be changed.
|
||||
|
||||
### Future Work
|
||||
|
||||
We are actively working on improving BlazePose GHUM 3D's Z prediction. It will
|
||||
allow us to use joint angles in the feature vectors, which are more natural and
|
||||
easier to configure (although distances can still be useful to detect touches
|
||||
between body parts) and to perform rotation normalization of poses and reduce
|
||||
the number of camera angles required for accurate k-NN classification.
|
||||
|
||||
## Resources
|
||||
|
||||
* Google AI Blog:
|
||||
@@ -387,7 +510,7 @@ on how to build MediaPipe examples.
|
||||
[BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
|
||||
([presentation](https://youtu.be/YPpUOTRn5tA))
|
||||
* [Models and model cards](./models.md#pose)
|
||||
|
||||
[Colab]:https://mediapipe.page.link/pose_py_colab
|
||||
|
||||
[web demo]:https://code.mediapipe.dev/codepen/pose
|
||||
* [Web demo](https://code.mediapipe.dev/codepen/pose)
|
||||
* [Python Colab](https://mediapipe.page.link/pose_py_colab)
|
||||
* [Pose Classification Colab (Basic)](https://mediapipe.page.link/pose_classification_basic)
|
||||
* [Pose Classification Colab (Extended)](https://mediapipe.page.link/pose_classification_extended)
|
||||
|
||||
@@ -18,7 +18,7 @@ has_toc: false
|
||||
|
||||
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
|
||||
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
|
||||
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
|
||||
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
@@ -28,7 +28,7 @@ has_toc: false
|
||||
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
|
||||
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
|
||||
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | | |
|
||||
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | | ✅ | |
|
||||
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
|
||||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
|
||||
@@ -37,7 +37,7 @@ The graph can be modified by adding and editing code in the Editor view.
|
||||
|
||||
|
||||
* Pressing the "Upload" button will prompt the user to select a local PBTXT
|
||||
file, which will everwrite the current code within the editor.
|
||||
file, which will overwrite the current code within the editor.
|
||||
|
||||
* Alternatively, code can be pasted directly into the editor window.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user