talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 612e50bb8db2ec3dc1c30049372d87a80c3848db
This commit is contained in:
MediaPipe Team
2020-08-30 19:52:55 -04:00
committed by chuoling
parent a7225b938a
commit c0124fb83c
248 changed files with 5225 additions and 1914 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/python/BUILD
+12 -9
View File
@@ -27,15 +27,18 @@ cc_library(
pybind_extension(
name = "_framework_bindings",
srcs = ["framework_bindings.cc"],
linkopts = [
"-lopencv_core",
"-lopencv_imgproc",
"-lopencv_highgui",
"-lopencv_video",
"-lopencv_features2d",
"-lopencv_calib3d",
"-lopencv_imgcodecs",
],
linkopts = select({
"//third_party:opencv_source_build": [],
"//conditions:default": [
"-lopencv_core",
"-lopencv_imgproc",
"-lopencv_highgui",
"-lopencv_video",
"-lopencv_features2d",
"-lopencv_calib3d",
"-lopencv_imgcodecs",
],
}),
deps = [
":builtin_calculators",
"//mediapipe/python/pybind:calculator_graph",
mediapipe/python/calculator_graph_test.py
+1 -1
View File
@@ -18,7 +18,7 @@
# Dependency imports
from absl.testing import absltest
import mediapipe.python as mp
import mediapipe as mp
from google.protobuf import text_format
from mediapipe.framework import calculator_pb2
mediapipe/python/image_frame_test.py
+1 -1
View File
@@ -17,9 +17,9 @@
import random
from absl.testing import absltest
import cv2
import mediapipe as mp
import numpy as np
import PIL.Image
import mediapipe.python as mp
# TODO: Add unit tests specifically for memory management.
mediapipe/python/packet_creator.py
+57 -11
View File
@@ -16,6 +16,7 @@
"""The public facing packet creator APIs."""
from typing import List, Union
import warnings
import numpy as np
@@ -48,34 +49,60 @@ create_string_to_packet_map = _packet_creator.create_string_to_packet_map
create_matrix = _packet_creator.create_matrix
def create_image_frame(
data: Union[image_frame.ImageFrame, np.ndarray],
*,
image_format: image_frame.ImageFormat = None) -> packet.Packet:
def create_image_frame(data: Union[image_frame.ImageFrame, np.ndarray],
*,
image_format: image_frame.ImageFormat = None,
copy: bool = None) -> packet.Packet:
"""Create a MediaPipe ImageFrame packet.
A MediaPipe ImageFrame packet can be created from either the raw pixel data
A MediaPipe ImageFrame packet can be created from an existing MediaPipe
ImageFrame object and the data will be realigned and copied into a new
ImageFrame object inside of the packet.
A MediaPipe ImageFrame packet can also be created from the raw pixel data
represented as a numpy array with one of the uint8, uint16, and float data
types or an existing MediaPipe ImageFrame object. The data will be realigned
and copied into an ImageFrame object inside of the packet.
types. There are three data ownership modes depending on how the 'copy' arg
is set.
i) Default mode
If copy is not set, mutable data is always copied while the immutable data
is by reference.
ii) Copy mode (safe)
If copy is set to True, the data will be realigned and copied into an
ImageFrame object inside of the packet regardless the immutablity of the
original data.
iii) Reference mode (dangerous)
If copy is set to False, the data will be forced to be shared. If the data is
mutable (data.flags.writeable is True), a warning will be raised.
Args:
data: A MediaPipe ImageFrame object or the raw pixel data that is
represnted as a numpy ndarray.
image_format: One of the image_frame.ImageFormat enum types.
copy: Indicate if the packet should copy the data from the numpy nparray.
Returns:
A MediaPipe ImageFrame Packet.
Raises:
ValueError:
i) When "data" is a numpy ndarray, "image_format" is not provided.
i) When "data" is a numpy ndarray, "image_format" is not provided or
the "data" array is not c_contiguous in the reference mode.
ii) When "data" is an ImageFrame object, the "image_format" arg doesn't
match the image format of the "data" ImageFrame object.
match the image format of the "data" ImageFrame object or "copy" is
explicitly set to False.
TypeError: If "image format" doesn't match "data" array's data type.
Examples:
np_array = np.random.randint(255, size=(321, 123, 3), dtype=np.uint8)
# Copy mode by default if the data array is writable.
image_frame_packet = mp.packet_creator.create_image_frame(
image_format=mp.ImageFormat.SRGB, data=np_array)
# Make the array unwriteable to trigger the reference mode.
np_array.flags.writeable = False
image_frame_packet = mp.packet_creator.create_image_frame(
image_format=mp.ImageFormat.SRGB, data=np_array)
@@ -87,14 +114,33 @@ def create_image_frame(
if image_format is not None and data.image_format != image_format:
raise ValueError(
'The provided image_format doesn\'t match the one from the data arg.')
if copy is not None and not copy:
raise ValueError(
'Creating image frame packet by taking a reference of another image frame object is not supported yet.'
)
# pylint:disable=protected-access
return _packet_creator._create_image_frame_with_copy(data)
return _packet_creator._create_image_frame_from_image_frame(data)
# pylint:enable=protected-access
else:
if image_format is None:
raise ValueError('Please provide \'image_format\' with \'data\'.')
# If copy arg is not set, copying the data if it's immutable. Otherwise,
# take a reference of the immutable data to avoid data copy.
if copy is None:
copy = True if data.flags.writeable else False
if not copy:
# TODO: Investigate why the first 2 bytes of the data has data
# corruption when "data" is not c_contiguous.
if not data.flags.c_contiguous:
raise ValueError(
'Reference mode is unavailable if \'data\' is not c_contiguous.')
if data.flags.writeable:
warnings.warn(
'\'data\' is still writeable. Taking a reference of the data to create ImageFrame packet is dangerous.',
RuntimeWarning, 2)
# pylint:disable=protected-access
return _packet_creator._create_image_frame_with_copy(image_format, data)
return _packet_creator._create_image_frame_from_pixel_data(
image_format, data, copy)
# pylint:enable=protected-access
mediapipe/python/packet_test.py
+46 -2
View File
@@ -18,8 +18,8 @@ import gc
import random
import sys
from absl.testing import absltest
import mediapipe as mp
import numpy as np
import mediapipe.python as mp
from google.protobuf import text_format
from mediapipe.framework.formats import detection_pb2
@@ -294,7 +294,51 @@ class PacketTest(absltest.TestCase):
# copy mode.
self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
def testImageFramePacketCopyConstuctionWithCropping(self):
def testImageFramePacketCreationReferenceMode(self):
w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
rgb_data = np.random.randint(255, size=(h, w, channels), dtype=np.uint8)
rgb_data.flags.writeable = False
initial_ref_count = sys.getrefcount(rgb_data)
image_frame_packet = mp.packet_creator.create_image_frame(
image_format=mp.ImageFormat.SRGB, data=rgb_data)
# Reference mode increase the ref count of the rgb_data by 1.
self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count + 1)
del image_frame_packet
gc.collect()
# Deleting image_frame_packet should decrese the ref count of rgb_data by 1.
self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
rgb_data_copy = np.copy(rgb_data)
# rgb_data_copy is a copy of rgb_data and should not increase the ref count.
self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
text_config = """
node {
calculator: 'PassThroughCalculator'
input_side_packet: "in"
output_side_packet: "out"
}
"""
graph = mp.CalculatorGraph(graph_config=text_config)
graph.start_run(
input_side_packets={
'in':
mp.packet_creator.create_image_frame(
image_format=mp.ImageFormat.SRGB, data=rgb_data)
})
# reference mode increase the ref count of the rgb_data by 1.
self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count + 1)
graph.wait_until_done()
output_packet = graph.get_output_side_packet('out')
del rgb_data
del graph
gc.collect()
# The pixel data of the output image frame packet should still be valid
# after the graph and the original rgb_data data are deleted.
self.assertTrue(
np.array_equal(
mp.packet_getter.get_image_frame(output_packet).numpy_view(),
rgb_data_copy))
def testImageFramePacketCopyCreationWithCropping(self):
w, h, channels = random.randrange(40, 100), random.randrange(40, 100), 3
channels, offset = 3, 10
rgb_data = np.random.randint(255, size=(h, w, channels), dtype=np.uint8)
mediapipe/python/pybind/BUILD
+1 -1
View File
@@ -14,7 +14,7 @@
load("@pybind11_bazel//:build_defs.bzl", "pybind_library")
licenses(["notice"]) # Apache 2.0
licenses(["notice"])
package(default_visibility = ["//mediapipe/python:__subpackages__"])
mediapipe/python/pybind/image_frame_util.h
+17 -15
View File
@@ -28,31 +28,33 @@ namespace python {
namespace py = pybind11;
// TODO: Implement the reference mode of image frame creation, which
// takes a reference to the external data rather than copying it over.
// A possible solution is to have a custom PixelDataDeleter:
// The refcount of the numpy array will be increased when the image frame is
// created by taking a reference to the external numpy array data. Then, the
// custom PixelDataDeleter will decrease the refcount when the image frame gets
// destroyed and let Python GC does its job.
template <typename T>
std::unique_ptr<ImageFrame> CreateImageFrame(
mediapipe::ImageFormat::Format format,
const py::array_t<T, py::array::c_style>& data) {
const py::array_t<T, py::array::c_style>& data, bool copy = true) {
int rows = data.shape()[0];
int cols = data.shape()[1];
int width_step = ImageFrame::NumberOfChannelsForFormat(format) *
ImageFrame::ByteDepthForFormat(format) * cols;
if (copy) {
auto image_frame = absl::make_unique<ImageFrame>(
format, /*width=*/cols, /*height=*/rows, width_step,
static_cast<uint8*>(data.request().ptr),
ImageFrame::PixelDataDeleter::kNone);
auto image_frame_copy = absl::make_unique<ImageFrame>();
// Set alignment_boundary to kGlDefaultAlignmentBoundary so that both
// GPU and CPU can process it.
image_frame_copy->CopyFrom(*image_frame,
ImageFrame::kGlDefaultAlignmentBoundary);
return image_frame_copy;
}
PyObject* data_pyobject = data.ptr();
auto image_frame = absl::make_unique<ImageFrame>(
format, /*width=*/cols, /*height=*/rows, width_step,
static_cast<uint8*>(data.request().ptr),
ImageFrame::PixelDataDeleter::kNone);
auto image_frame_copy = absl::make_unique<ImageFrame>();
// Set alignment_boundary to kGlDefaultAlignmentBoundary so that both
// GPU and CPU can process it.
image_frame_copy->CopyFrom(*image_frame,
ImageFrame::kGlDefaultAlignmentBoundary);
return image_frame_copy;
/*deleter=*/[data_pyobject](uint8*) { Py_XDECREF(data_pyobject); });
Py_XINCREF(data_pyobject);
return image_frame;
}
} // namespace python
mediapipe/python/pybind/packet_creator.cc
+9 -32
View File
@@ -31,18 +31,18 @@ namespace python {
namespace {
Packet CreateImageFramePacket(mediapipe::ImageFormat::Format format,
const py::array& data) {
const py::array& data, bool copy) {
if (format == mediapipe::ImageFormat::SRGB ||
format == mediapipe::ImageFormat::SRGBA ||
format == mediapipe::ImageFormat::GRAY8) {
return Adopt(CreateImageFrame<uint8>(format, data).release());
return Adopt(CreateImageFrame<uint8>(format, data, copy).release());
} else if (format == mediapipe::ImageFormat::GRAY16 ||
format == mediapipe::ImageFormat::SRGB48 ||
format == mediapipe::ImageFormat::SRGBA64) {
return Adopt(CreateImageFrame<uint16>(format, data).release());
return Adopt(CreateImageFrame<uint16>(format, data, copy).release());
} else if (format == mediapipe::ImageFormat::VEC32F1 ||
format == mediapipe::ImageFormat::VEC32F2) {
return Adopt(CreateImageFrame<float>(format, data).release());
return Adopt(CreateImageFrame<float>(format, data, copy).release());
}
throw RaisePyError(PyExc_RuntimeError,
absl::StrCat("Unsupported ImageFormat: ", format).c_str());
@@ -560,26 +560,12 @@ void PublicPacketCreators(pybind11::module* m) {
}
void InternalPacketCreators(pybind11::module* m) {
m->def(
"_create_image_frame_with_copy",
[](mediapipe::ImageFormat::Format format, const py::array& data) {
return CreateImageFramePacket(format, data);
},
py::arg("format"), py::arg("data").noconvert(),
py::return_value_policy::move);
m->def("_create_image_frame_from_pixel_data", &CreateImageFramePacket,
py::arg("format"), py::arg("data").noconvert(), py::arg("copy"),
py::return_value_policy::move);
m->def(
"_create_image_frame_with_reference",
[](mediapipe::ImageFormat::Format format, const py::array& data) {
throw RaisePyError(
PyExc_NotImplementedError,
"Creating image frame packet with reference is not supproted yet.");
},
py::arg("format"), py::arg("data").noconvert(),
py::return_value_policy::move);
m->def(
"_create_image_frame_with_copy",
"_create_image_frame_from_image_frame",
[](ImageFrame& image_frame) {
auto image_frame_copy = absl::make_unique<ImageFrame>();
// Set alignment_boundary to kGlDefaultAlignmentBoundary so that
@@ -590,15 +576,6 @@ void InternalPacketCreators(pybind11::module* m) {
},
py::arg("image_frame").noconvert(), py::return_value_policy::move);
m->def(
"_create_image_frame_with_reference",
[](ImageFrame& image_frame) {
throw RaisePyError(
PyExc_NotImplementedError,
"Creating image frame packet with reference is not supproted yet.");
},
py::arg("image_frame").noconvert(), py::return_value_policy::move);
m->def(
"_create_proto",
[](const std::string& type_name, const py::bytes& serialized_proto) {
@@ -616,7 +593,7 @@ void InternalPacketCreators(pybind11::module* m) {
std::move(maybe_holder).ValueOrDie();
auto* copy = const_cast<proto_ns::MessageLite*>(
message_holder->GetProtoMessageLite());
copy->ParseFromString(serialized_proto);
copy->ParseFromString(std::string(serialized_proto));
return packet_internal::Create(message_holder.release());
},
py::return_value_policy::move);
mediapipe/python/timestamp_test.py
+1 -1
View File
@@ -17,7 +17,7 @@
import time
from absl.testing import absltest
import mediapipe.python as mp
import mediapipe as mp
class TimestampTest(absltest.TestCase):