talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: d8caa66de45839696f5bd0786ad3bfbcb9cff632
This commit is contained in:
MediaPipe Team
2020-12-09 22:43:33 -05:00
committed by chuoling
parent f15da632de
commit 2b58cceec9
750 changed files with 22901 additions and 9478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/python/BUILD
+8 -2
View File
@@ -53,14 +53,20 @@ pybind_extension(
cc_library(
name = "builtin_calculators",
deps = [
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:pass_through_calculator",
"//mediapipe/calculators/core:side_packet_to_stream_calculator",
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:string_to_int_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/util:detection_unique_id_calculator",
"//mediapipe/modules/face_detection:face_detection_front_cpu",
"//mediapipe/modules/face_landmark:face_landmark_front_cpu",
"//mediapipe/modules/hand_landmark:hand_landmark_tracking_cpu",
"//mediapipe/modules/holistic_landmark:holistic_landmark_cpu",
"//mediapipe/modules/palm_detection:palm_detection_cpu",
"//mediapipe/modules/pose_detection:pose_detection_cpu",
"//mediapipe/modules/pose_landmark:pose_landmark_upper_body_by_roi_cpu",
"//mediapipe/modules/pose_landmark:pose_landmark_upper_body_smoothed_cpu",
"//mediapipe/modules/pose_landmark:pose_landmark_by_roi_cpu",
"//mediapipe/modules/pose_landmark:pose_landmark_cpu",
],
)
mediapipe/python/calculator_graph_test.py
+11 -10
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""Tests for mediapipe.python._framework_bindings.calculator_graph."""
# Dependency imports
@@ -25,11 +24,13 @@ from mediapipe.framework import calculator_pb2
class GraphTest(absltest.TestCase):
def testInvalidBinaryGraphFile(self):
with self.assertRaisesRegex(FileNotFoundError, 'No such file or directory'):
def test_invalid_binary_graph_file(self):
with self.assertRaisesRegex(
FileNotFoundError,
'(No such file or directory|The path does not exist)'):
mp.CalculatorGraph(binary_graph_path='/tmp/abc.binarypb')
def testInvalidNodeConfig(self):
def test_invalid_node_config(self):
text_config = """
node {
calculator: 'PassThroughCalculator'
@@ -46,7 +47,7 @@ class GraphTest(absltest.TestCase):
):
mp.CalculatorGraph(graph_config=config_proto)
def testInvalidCalculatorType(self):
def test_invalid_calculator_type(self):
text_config = """
node {
calculator: 'SomeUnknownCalculator'
@@ -60,7 +61,7 @@ class GraphTest(absltest.TestCase):
RuntimeError, 'Unable to find Calculator \"SomeUnknownCalculator\"'):
mp.CalculatorGraph(graph_config=config_proto)
def testGraphInitializedWithProtoConfig(self):
def test_graph_initialized_with_proto_config(self):
text_config = """
max_queue_size: 1
input_stream: 'in'
@@ -95,7 +96,7 @@ class GraphTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_str(out[0]), 'hello world')
self.assertEqual(mp.packet_getter.get_str(out[1]), 'hello world')
def testGraphInitializedWithTextConfig(self):
def test_graph_initialized_with_text_config(self):
text_config = """
max_queue_size: 1
input_stream: 'in'
@@ -127,7 +128,7 @@ class GraphTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_str(out[0]), 'hello world')
self.assertEqual(mp.packet_getter.get_str(out[1]), 'hello world')
def testGraphValidationAndInitialization(self):
def test_graph_validation_and_initialization(self):
text_config = """
max_queue_size: 1
input_stream: 'in'
@@ -164,7 +165,7 @@ class GraphTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_str(out[0]), 'hello world')
self.assertEqual(mp.packet_getter.get_str(out[1]), 'hello world')
def testInsertPacketsWithSameTimestamp(self):
def test_insert_packets_with_same_timestamp(self):
text_config = """
max_queue_size: 1
input_stream: 'in'
@@ -192,7 +193,7 @@ class GraphTest(absltest.TestCase):
ValueError, 'Current minimum expected timestamp is 1 but received 0.'):
graph.wait_until_idle()
def testSidePacketGraph(self):
def test_side_packet_graph(self):
text_config = """
node {
calculator: 'StringToUint64Calculator'
mediapipe/python/image_frame_test.py
+11 -11
View File
@@ -27,7 +27,7 @@ import PIL.Image
# TODO: Add unit tests specifically for memory management.
class ImageFrameTest(absltest.TestCase):
def testCreateImageFrameFromGrayCvMat(self):
def test_create_image_frame_from_gray_cv_mat(self):
w, h = random.randrange(3, 100), random.randrange(3, 100)
mat = cv2.cvtColor(
np.random.randint(2**8 - 1, size=(h, w, 3), dtype=np.uint8),
@@ -41,7 +41,7 @@ class ImageFrameTest(absltest.TestCase):
print(image_frame[w, h])
self.assertEqual(42, image_frame[2, 2])
def testCreateImageFrameFromRgbCvMat(self):
def test_create_image_frame_from_rgb_cv_mat(self):
w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
mat = cv2.cvtColor(
np.random.randint(2**8 - 1, size=(h, w, channels), dtype=np.uint8),
@@ -53,7 +53,7 @@ class ImageFrameTest(absltest.TestCase):
print(image_frame[w, h, channels])
self.assertEqual(42, image_frame[2, 2, 1])
def testCreateImageFrameFromRgb48CvMat(self):
def test_create_image_frame_from_rgb48_cv_mat(self):
w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
mat = cv2.cvtColor(
np.random.randint(2**16 - 1, size=(h, w, channels), dtype=np.uint16),
@@ -65,7 +65,7 @@ class ImageFrameTest(absltest.TestCase):
print(image_frame[w, h, channels])
self.assertEqual(42, image_frame[2, 2, 1])
def testCreateImageFrameFromGrayPilImage(self):
def test_create_image_frame_from_gray_pil_image(self):
w, h = random.randrange(3, 100), random.randrange(3, 100)
img = PIL.Image.fromarray(
np.random.randint(2**8 - 1, size=(h, w), dtype=np.uint8), 'L')
@@ -77,7 +77,7 @@ class ImageFrameTest(absltest.TestCase):
with self.assertRaisesRegex(IndexError, 'out of bounds'):
print(image_frame[w, h])
def testCreateImageFrameFromRgbPilImage(self):
def test_create_image_frame_from_rgb_pil_image(self):
w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
img = PIL.Image.fromarray(
np.random.randint(2**8 - 1, size=(h, w, channels), dtype=np.uint8),
@@ -88,7 +88,7 @@ class ImageFrameTest(absltest.TestCase):
with self.assertRaisesRegex(IndexError, 'out of bounds'):
print(image_frame[w, h, channels])
def testCreateImageFrameFromRgba64PilImage(self):
def test_create_image_frame_from_rgba64_pil_image(self):
w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 4
img = PIL.Image.fromarray(
np.random.randint(2**16 - 1, size=(h, w, channels), dtype=np.uint16),
@@ -100,7 +100,7 @@ class ImageFrameTest(absltest.TestCase):
with self.assertRaisesRegex(IndexError, 'out of bounds'):
print(image_frame[1000, 1000, 1000])
def testImageFrameNumbyView(self):
def test_image_frame_numby_view(self):
w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
mat = cv2.cvtColor(
np.random.randint(2**8 - 1, size=(h, w, channels), dtype=np.uint8),
@@ -116,7 +116,7 @@ class ImageFrameTest(absltest.TestCase):
copied_ndarray = np.copy(output_ndarray)
copied_ndarray[0, 0, 0] = 0
def testCroppedGray8Image(self):
def test_cropped_gray8_image(self):
w, h = random.randrange(20, 100), random.randrange(20, 100)
channels, offset = 3, 10
mat = cv2.cvtColor(
@@ -129,7 +129,7 @@ class ImageFrameTest(absltest.TestCase):
np.array_equal(mat[offset:-offset, offset:-offset],
image_frame.numpy_view()))
def testCroppedRGBImage(self):
def test_cropped_rgb_image(self):
w, h = random.randrange(20, 100), random.randrange(20, 100)
channels, offset = 3, 10
mat = cv2.cvtColor(
@@ -145,7 +145,7 @@ class ImageFrameTest(absltest.TestCase):
# For image frames that store contiguous data, the output of numpy_view()
# points to the pixel data of the original image frame object. The life cycle
# of the data array should tie to the image frame object.
def testImageFrameNumpyViewWithContiguousData(self):
def test_image_frame_numpy_view_with_contiguous_data(self):
w, h = 640, 480
mat = np.random.randint(2**8 - 1, size=(h, w, 3), dtype=np.uint8)
image_frame = mp.ImageFrame(image_format=mp.ImageFormat.SRGB, data=mat)
@@ -168,7 +168,7 @@ class ImageFrameTest(absltest.TestCase):
# For image frames that store non contiguous data, the output of numpy_view()
# stores a copy of the pixel data of the image frame object. The life cycle of
# the data array doesn't tie to the image frame object.
def testImageFrameNumpyViewWithNonContiguousData(self):
def test_image_frame_numpy_view_with_non_contiguous_data(self):
w, h = 641, 481
mat = np.random.randint(2**8 - 1, size=(h, w, 3), dtype=np.uint8)
image_frame = mp.ImageFrame(image_format=mp.ImageFormat.SRGB, data=mat)
mediapipe/python/packet_creator.py
+1 -1
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""The public facing packet creator APIs."""
from typing import List, Union
@@ -42,6 +41,7 @@ create_double = _packet_creator.create_double
create_int_array = _packet_creator.create_int_array
create_float_array = _packet_creator.create_float_array
create_int_vector = _packet_creator.create_int_vector
create_bool_vector = _packet_creator.create_bool_vector
create_float_vector = _packet_creator.create_float_vector
create_string_vector = _packet_creator.create_string_vector
create_packet_vector = _packet_creator.create_packet_vector
mediapipe/python/packet_getter.py
+1 -1
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""The public facing packet getter APIs."""
from typing import List, Type
@@ -29,6 +28,7 @@ get_int = _packet_getter.get_int
get_uint = _packet_getter.get_uint
get_float = _packet_getter.get_float
get_int_list = _packet_getter.get_int_list
get_bool_list = _packet_getter.get_bool_list
get_float_list = _packet_getter.get_float_list
get_str_list = _packet_getter.get_str_list
get_packet_list = _packet_getter.get_packet_list
mediapipe/python/packet_test.py
+31 -31
View File
@@ -26,17 +26,17 @@ from mediapipe.framework.formats import detection_pb2
class PacketTest(absltest.TestCase):
def testEmptyPacket(self):
def test_empty_packet(self):
p = mp.Packet()
self.assertTrue(p.is_empty())
def testBooleanPacket(self):
def test_boolean_packet(self):
p = mp.packet_creator.create_bool(True)
p.timestamp = 0
self.assertEqual(mp.packet_getter.get_bool(p), True)
self.assertEqual(p.timestamp, 0)
def testIntPacket(self):
def test_int_packet(self):
with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
p = mp.packet_creator.create_int(2**32)
p = mp.packet_creator.create_int(42)
@@ -48,7 +48,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_int(p2), 1)
self.assertEqual(p2.timestamp, 0)
def testInt8Packet(self):
def test_int8_packet(self):
with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
p = mp.packet_creator.create_int8(2**7)
p = mp.packet_creator.create_int8(2**7 - 1)
@@ -60,7 +60,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_int(p2), 1)
self.assertEqual(p2.timestamp, 0)
def testInt16Packet(self):
def test_int16_packet(self):
with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
p = mp.packet_creator.create_int16(2**15)
p = mp.packet_creator.create_int16(2**15 - 1)
@@ -72,7 +72,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_int(p2), 1)
self.assertEqual(p2.timestamp, 0)
def testInt32Packet(self):
def test_int32_packet(self):
with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
p = mp.packet_creator.create_int32(2**31)
@@ -85,7 +85,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_int(p2), 1)
self.assertEqual(p2.timestamp, 0)
def testInt64Packet(self):
def test_int64_packet(self):
p = mp.packet_creator.create_int64(2**63 - 1)
p.timestamp = 0
self.assertEqual(mp.packet_getter.get_int(p), 2**63 - 1)
@@ -95,7 +95,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_int(p2), 1)
self.assertEqual(p2.timestamp, 0)
def testUint8Packet(self):
def test_uint8_packet(self):
with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
p = mp.packet_creator.create_uint8(2**8)
p = mp.packet_creator.create_uint8(2**8 - 1)
@@ -107,7 +107,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_uint(p2), 1)
self.assertEqual(p2.timestamp, 0)
def testUint16Packet(self):
def test_uint16_packet(self):
with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
p = mp.packet_creator.create_uint16(2**16)
p = mp.packet_creator.create_uint16(2**16 - 1)
@@ -119,7 +119,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_uint(p2), 1)
self.assertEqual(p2.timestamp, 0)
def testUint32Packet(self):
def test_uint32_packet(self):
with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
p = mp.packet_creator.create_uint32(2**32)
p = mp.packet_creator.create_uint32(2**32 - 1)
@@ -131,7 +131,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_uint(p2), 1)
self.assertEqual(p2.timestamp, 0)
def testUint64Packet(self):
def test_uint64_packet(self):
p = mp.packet_creator.create_uint64(2**64 - 1)
p.timestamp = 0
self.assertEqual(mp.packet_getter.get_uint(p), 2**64 - 1)
@@ -141,7 +141,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_uint(p2), 1)
self.assertEqual(p2.timestamp, 0)
def testFloatPacket(self):
def test_float_packet(self):
p = mp.packet_creator.create_float(0.42)
p.timestamp = 0
self.assertAlmostEqual(mp.packet_getter.get_float(p), 0.42)
@@ -151,7 +151,7 @@ class PacketTest(absltest.TestCase):
self.assertAlmostEqual(mp.packet_getter.get_float(p2), 0.42)
self.assertEqual(p2.timestamp, 0)
def testDoublePacket(self):
def test_double_packet(self):
p = mp.packet_creator.create_double(0.42)
p.timestamp = 0
self.assertAlmostEqual(mp.packet_getter.get_float(p), 0.42)
@@ -161,37 +161,37 @@ class PacketTest(absltest.TestCase):
self.assertAlmostEqual(mp.packet_getter.get_float(p2), 0.42)
self.assertEqual(p2.timestamp, 0)
def testDetectionProtoPacket(self):
def test_detection_proto_packet(self):
detection = detection_pb2.Detection()
text_format.Parse('score: 0.5', detection)
p = mp.packet_creator.create_proto(detection).at(100)
def testStringPacket(self):
def test_string_packet(self):
p = mp.packet_creator.create_string('abc').at(100)
self.assertEqual(mp.packet_getter.get_str(p), 'abc')
self.assertEqual(p.timestamp, 100)
p.timestamp = 200
self.assertEqual(p.timestamp, 200)
def testBytesPacket(self):
def test_bytes_packet(self):
p = mp.packet_creator.create_string(b'xd0\xba\xd0').at(300)
self.assertEqual(mp.packet_getter.get_bytes(p), b'xd0\xba\xd0')
self.assertEqual(p.timestamp, 300)
def testIntArrayPacket(self):
def test_int_array_packet(self):
p = mp.packet_creator.create_int_array([1, 2, 3]).at(100)
self.assertEqual(p.timestamp, 100)
def testFloatArrayPacket(self):
def test_float_array_packet(self):
p = mp.packet_creator.create_float_array([0.1, 0.2, 0.3]).at(100)
self.assertEqual(p.timestamp, 100)
def testIntVectorPacket(self):
def test_int_vector_packet(self):
p = mp.packet_creator.create_int_vector([1, 2, 3]).at(100)
self.assertEqual(mp.packet_getter.get_int_list(p), [1, 2, 3])
self.assertEqual(p.timestamp, 100)
def testFloatVectorPacket(self):
def test_float_vector_packet(self):
p = mp.packet_creator.create_float_vector([0.1, 0.2, 0.3]).at(100)
output_list = mp.packet_getter.get_float_list(p)
self.assertAlmostEqual(output_list[0], 0.1)
@@ -199,7 +199,7 @@ class PacketTest(absltest.TestCase):
self.assertAlmostEqual(output_list[2], 0.3)
self.assertEqual(p.timestamp, 100)
def testStringVectorPacket(self):
def test_string_vector_packet(self):
p = mp.packet_creator.create_string_vector(['a', 'b', 'c']).at(100)
output_list = mp.packet_getter.get_str_list(p)
self.assertEqual(output_list[0], 'a')
@@ -207,7 +207,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(output_list[2], 'c')
self.assertEqual(p.timestamp, 100)
def testPacketVectorPacket(self):
def test_packet_vector_packet(self):
p = mp.packet_creator.create_packet_vector([
mp.packet_creator.create_float(0.42),
mp.packet_creator.create_int(42),
@@ -219,7 +219,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_str(output_list[2]), '42')
self.assertEqual(p.timestamp, 100)
def testStringToPacketMapPacket(self):
def test_string_to_packet_map_packet(self):
p = mp.packet_creator.create_string_to_packet_map({
'float': mp.packet_creator.create_float(0.42),
'int': mp.packet_creator.create_int(42),
@@ -232,7 +232,7 @@ class PacketTest(absltest.TestCase):
self.assertEqual(mp.packet_getter.get_str(output_list['string']), '42')
self.assertEqual(p.timestamp, 100)
def testUint8ImageFramePacket(self):
def test_uint8_image_frame_packet(self):
uint8_img = np.random.randint(
2**8 - 1,
size=(random.randrange(3, 100), random.randrange(3, 100), 3),
@@ -242,7 +242,7 @@ class PacketTest(absltest.TestCase):
output_image_frame = mp.packet_getter.get_image_frame(p)
self.assertTrue(np.array_equal(output_image_frame.numpy_view(), uint8_img))
def testUint16ImageFramePacket(self):
def test_uint16_image_frame_packet(self):
uint16_img = np.random.randint(
2**16 - 1,
size=(random.randrange(3, 100), random.randrange(3, 100), 4),
@@ -252,7 +252,7 @@ class PacketTest(absltest.TestCase):
output_image_frame = mp.packet_getter.get_image_frame(p)
self.assertTrue(np.array_equal(output_image_frame.numpy_view(), uint16_img))
def testFloatImageFramePacket(self):
def test_float_image_frame_packet(self):
float_img = np.float32(
np.random.random_sample(
(random.randrange(3, 100), random.randrange(3, 100), 2)))
@@ -261,7 +261,7 @@ class PacketTest(absltest.TestCase):
output_image_frame = mp.packet_getter.get_image_frame(p)
self.assertTrue(np.allclose(output_image_frame.numpy_view(), float_img))
def testImageFramePacketCreationCopyMode(self):
def test_image_frame_packet_creation_copy_mode(self):
w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
rgb_data = np.random.randint(255, size=(h, w, channels), dtype=np.uint8)
# rgb_data is c_contiguous.
@@ -294,7 +294,7 @@ class PacketTest(absltest.TestCase):
# copy mode.
self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
def testImageFramePacketCreationReferenceMode(self):
def test_image_frame_packet_creation_reference_mode(self):
w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
rgb_data = np.random.randint(255, size=(h, w, channels), dtype=np.uint8)
rgb_data.flags.writeable = False
@@ -338,7 +338,7 @@ class PacketTest(absltest.TestCase):
mp.packet_getter.get_image_frame(output_packet).numpy_view(),
rgb_data_copy))
def testImageFramePacketCopyCreationWithCropping(self):
def test_image_frame_packet_copy_creation_with_cropping(self):
w, h, channels = random.randrange(40, 100), random.randrange(40, 100), 3
channels, offset = 3, 10
rgb_data = np.random.randint(255, size=(h, w, channels), dtype=np.uint8)
@@ -362,7 +362,7 @@ class PacketTest(absltest.TestCase):
# copy mode.
self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
def testMatrixPacket(self):
def test_matrix_packet(self):
np_matrix = np.array([[.1, .2, .3], [.4, .5, .6]])
initial_ref_count = sys.getrefcount(np_matrix)
p = mp.packet_creator.create_matrix(np_matrix)
@@ -374,7 +374,7 @@ class PacketTest(absltest.TestCase):
self.assertTrue(
np.allclose(output_matrix, np.array([[.1, .2, .3], [.4, .5, .6]])))
def testMatrixPacketWithNonCContiguousData(self):
def test_matrix_packet_with_non_c_contiguous_data(self):
np_matrix = np.array([[.1, .2, .3], [.4, .5, .6]])[:, ::-1]
# np_matrix is not c_contiguous.
self.assertFalse(np_matrix.flags.c_contiguous)
mediapipe/python/pybind/calculator_graph.cc
+2 -2
View File
@@ -284,7 +284,7 @@ void CalculatorGraphSubmodule(pybind11::module* module) {
graph.close()
)doc",
py::arg("input_side_packets") = (py::dict){});
py::arg("input_side_packets") = py::dict());
calculator_graph.def(
"wait_until_done",
@@ -376,7 +376,7 @@ void CalculatorGraphSubmodule(pybind11::module* module) {
calculator_graph.def(
"get_combined_error_message",
[](CalculatorGraph* self) {
::mediapipe::Status error_status;
mediapipe::Status error_status;
if (self->GetCombinedErrors(&error_status) && !error_status.ok()) {
return error_status.ToString();
}
mediapipe/python/pybind/image_frame.cc
+1 -1
View File
@@ -367,5 +367,5 @@ void ImageFrameSubmodule(pybind11::module* module) {
} // namespace mediapipe
#include "mediapipe/framework/type_map.h"
MEDIAPIPE_REGISTER_TYPE(::mediapipe::ImageFrame, "::mediapipe::ImageFrame",
MEDIAPIPE_REGISTER_TYPE(mediapipe::ImageFrame, "::mediapipe::ImageFrame",
nullptr, nullptr);
mediapipe/python/pybind/packet_creator.cc
+22
View File
@@ -426,6 +426,28 @@ void PublicPacketCreators(pybind11::module* m) {
)doc",
py::arg().noconvert(), py::return_value_policy::move);
m->def(
"create_bool_vector",
[](const std::vector<bool>& data) {
return MakePacket<std::vector<bool>>(data);
},
R"doc(Create a MediaPipe bool vector Packet from a list of booleans.
Args:
data: A list of booleans.
Returns:
A MediaPipe bool vector Packet.
Raises:
TypeError: If the input is not a list of booleans.
Examples:
packet = mp.packet_creator.create_bool_vector([True, True, False])
data = mp.packet_getter.get_bool_vector(packet)
)doc",
py::arg().noconvert(), py::return_value_policy::move);
m->def(
"create_float_vector",
[](const std::vector<float>& data) {
mediapipe/python/pybind/packet_getter.cc
+18
View File
@@ -205,6 +205,24 @@ void PublicPacketGetters(pybind11::module* m) {
data = mp.packet_getter.get_int_list(packet)
)doc");
m->def(
"get_bool_list", &GetContent<std::vector<bool>>,
R"doc(Get the content of a MediaPipe bool vector Packet as a boolean list.
Args:
packet: A MediaPipe Packet that holds std:vector<bool>.
Returns:
An boolean list.
Raises:
ValueError: If the Packet doesn't contain std:vector<bool>.
Examples:
packet = mp.packet_creator.create_bool_vector([True, True, False])
data = mp.packet_getter.get_bool_list(packet)
)doc");
m->def(
"get_float_list", &GetContent<std::vector<float>>,
R"doc(Get the content of a MediaPipe float vector Packet as a float list.
mediapipe/python/pybind/util.h
+2 -1
View File
@@ -97,7 +97,8 @@ inline ::mediapipe::CalculatorGraphConfig ReadCalculatorGraphConfigFromFile(
throw RaisePyError(PyExc_FileNotFoundError, status.message().data());
}
std::string graph_config_string;
RaisePyErrorIfNotOk(file::GetContents(file_name, &graph_config_string));
RaisePyErrorIfNotOk(file::GetContents(file_name, &graph_config_string,
/*read_as_binary=*/true));
if (!graph_config_proto.ParseFromArray(graph_config_string.c_str(),
graph_config_string.length())) {
throw RaisePyError(
mediapipe/python/solution_base.py
+11 -2
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""MediaPipe SolutionBase module.
MediaPipe SolutionBase is the common base class for the high-level MediaPipe
@@ -32,9 +31,12 @@ import numpy as np
from google.protobuf import descriptor
# resources dependency
# pylint: disable=unused-import
# pylint: enable=unused-import
from mediapipe.framework import calculator_pb2
# pylint: disable=unused-import
from mediapipe.framework.formats import detection_pb2
from mediapipe.calculators.core import constant_side_packet_calculator_pb2
from mediapipe.calculators.image import image_transformation_calculator_pb2
from mediapipe.calculators.tensor import tensors_to_detections_calculator_pb2
from mediapipe.calculators.util import landmarks_smoothing_calculator_pb2
@@ -55,6 +57,8 @@ import mediapipe.python.packet_getter as packet_getter
RGB_CHANNELS = 3
# TODO: Enable calculator options modification for more calculators.
CALCULATOR_TO_OPTIONS = {
'ConstantSidePacketCalculator':
constant_side_packet_calculator_pb2.ConstantSidePacketCalculatorOptions,
'ImageTransformationCalculator':
image_transformation_calculator_pb2
.ImageTransformationCalculatorOptions,
@@ -76,6 +80,7 @@ class _PacketDataType(enum.Enum):
"""The packet data types supported by the SolutionBase class."""
STRING = 'string'
BOOL = 'bool'
BOOL_LIST = 'bool_list'
INT = 'int'
FLOAT = 'float'
AUDIO = 'matrix'
@@ -93,6 +98,8 @@ NAME_TO_TYPE: Mapping[str, '_PacketDataType'] = {
_PacketDataType.STRING,
'bool':
_PacketDataType.BOOL,
'::std::vector<bool>':
_PacketDataType.BOOL_LIST,
'int':
_PacketDataType.INT,
'float':
@@ -113,6 +120,8 @@ NAME_TO_TYPE: Mapping[str, '_PacketDataType'] = {
_PacketDataType.PROTO,
'::mediapipe::NormalizedLandmark':
_PacketDataType.PROTO,
'::mediapipe::Trigger':
_PacketDataType.PROTO,
'::mediapipe::Rect':
_PacketDataType.PROTO,
'::mediapipe::NormalizedRect':
@@ -198,7 +207,7 @@ class SolutionBase:
raise ValueError(
"Must provide exactly one of 'binary_graph_path' or 'graph_config'.")
# MediaPipe package root path
root_path = os.sep.join( os.path.abspath(__file__).split(os.sep)[:-3])
root_path = os.sep.join(os.path.abspath(__file__).split(os.sep)[:-3])
resource_util.set_resource_dir(root_path)
validated_graph = validated_graph_config.ValidatedGraphConfig()
if binary_graph_path:
mediapipe/python/solution_base_test.py
-1
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""Tests for mediapipe.python.solution_base."""
from absl.testing import absltest
mediapipe/python/solutions/__init__.py
+1
View File
@@ -17,4 +17,5 @@
import mediapipe.python.solutions.drawing_utils
import mediapipe.python.solutions.face_mesh
import mediapipe.python.solutions.hands
import mediapipe.python.solutions.holistic
import mediapipe.python.solutions.pose
mediapipe/python/solutions/drawing_utils.py
+6 -2
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""MediaPipe solution drawing utils."""
import math
@@ -24,8 +23,10 @@ import numpy as np
from mediapipe.framework.formats import landmark_pb2
PRESENCE_THRESHOLD = 0.5
RGB_CHANNELS = 3
RED_COLOR = (0, 0, 255)
VISIBILITY_THRESHOLD = 0.5
@dataclasses.dataclass
@@ -88,7 +89,10 @@ def draw_landmarks(
image_rows, image_cols, _ = image.shape
idx_to_coordinates = {}
for idx, landmark in enumerate(landmark_list.landmark):
if landmark.visibility < 0 or landmark.presence < 0:
if ((landmark.HasField('visibility') and
landmark.visibility < VISIBILITY_THRESHOLD) or
(landmark.HasField('presence') and
landmark.presence < PRESENCE_THRESHOLD)):
continue
landmark_px = _normalized_to_pixel_coordinates(landmark.x, landmark.y,
image_cols, image_rows)
mediapipe/python/solutions/drawing_utils_test.py
+3 -4
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""Tests for mediapipe.python.solutions.drawing_utils."""
from absl.testing import absltest
@@ -48,7 +47,7 @@ class DrawingUtilTest(parameterized.TestCase):
@parameterized.named_parameters(
('landmark_list_has_only_one_element', 'landmark {x: 0.1 y: 0.1}'),
('second_landmark_is_invisible',
'landmark {x: 0.1 y: 0.1} landmark {x: 0.5 y: 0.5 visibility: -1.0}'))
'landmark {x: 0.1 y: 0.1} landmark {x: 0.5 y: 0.5 visibility: 0.0}'))
def test_draw_single_landmark_point(self, landmark_list_text):
landmark_list = text_format.Parse(landmark_list_text,
landmark_pb2.NormalizedLandmarkList())
@@ -65,8 +64,8 @@ class DrawingUtilTest(parameterized.TestCase):
('landmarks_have_x_and_y_only',
'landmark {x: 0.1 y: 0.5} landmark {x: 0.5 y: 0.1}'),
('landmark_zero_visibility_and_presence',
'landmark {x: 0.1 y: 0.5 presence: 0.0}'
'landmark {x: 0.5 y: 0.1 visibility: 0.0}'))
'landmark {x: 0.1 y: 0.5 presence: 0.5}'
'landmark {x: 0.5 y: 0.1 visibility: 0.5}'))
def test_draw_landmarks_and_connections(self, landmark_list_text):
landmark_list = text_format.Parse(landmark_list_text,
landmark_pb2.NormalizedLandmarkList())
mediapipe/python/solutions/face_mesh.py
+48 -117
View File
@@ -12,16 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""MediaPipe FaceMesh."""
from typing import NamedTuple
import numpy as np
from mediapipe.calculators.core import constant_side_packet_calculator_pb2
# pylint: disable=unused-import
from mediapipe.calculators.core import gate_calculator_pb2
from mediapipe.calculators.core import split_vector_calculator_pb2
from mediapipe.calculators.tensor import image_to_tensor_calculator_pb2
from mediapipe.calculators.tensor import inference_calculator_pb2
from mediapipe.calculators.tensor import tensors_to_classification_calculator_pb2
from mediapipe.calculators.tensor import tensors_to_detections_calculator_pb2
@@ -80,32 +81,6 @@ FACE_CONNECTIONS = frozenset([
(310, 415),
(415, 308),
# Left eye.
(33, 7),
(7, 163),
(163, 144),
(144, 145),
(145, 153),
(153, 154),
(154, 155),
(155, 133),
(33, 246),
(246, 161),
(161, 160),
(160, 159),
(159, 158),
(158, 157),
(157, 173),
(173, 133),
# Left eyebrow.
(46, 53),
(53, 52),
(52, 65),
(65, 55),
(70, 63),
(63, 105),
(105, 66),
(66, 107),
# Right eye.
(263, 249),
(249, 390),
(390, 373),
@@ -122,7 +97,7 @@ FACE_CONNECTIONS = frozenset([
(385, 384),
(384, 398),
(398, 362),
# Right eyebrow.
# Left eyebrow.
(276, 283),
(283, 282),
(282, 295),
@@ -131,6 +106,32 @@ FACE_CONNECTIONS = frozenset([
(293, 334),
(334, 296),
(296, 336),
# Right eye.
(33, 7),
(7, 163),
(163, 144),
(144, 145),
(145, 153),
(153, 154),
(154, 155),
(155, 133),
(33, 246),
(246, 161),
(161, 160),
(160, 159),
(159, 158),
(158, 157),
(157, 173),
(173, 133),
# Right eyebrow.
(46, 53),
(53, 52),
(52, 65),
(65, 55),
(70, 63),
(63, 105),
(105, 66),
(66, 107),
# Face oval.
(10, 338),
(338, 297),
@@ -177,111 +178,41 @@ class FaceMesh(SolutionBase):
MediaPipe FaceMesh processes an RGB image and returns the face landmarks on
each detected face.
Usage examples:
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_face_mesh = mp.solutions.face_mesh
# For static images:
face_mesh = mp_face_mesh.FaceMesh(
static_image_mode=True,
max_num_faces=1,
min_detection_confidence=0.5)
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
for idx, file in enumerate(file_list):
image = cv2.imread(file)
# Convert the BGR image to RGB before processing.
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
# Print and draw face mesh landmarks on the image.
if not results.multi_face_landmarks:
continue
annotated_image = image.copy()
for face_landmarks in results.multi_face_landmarks:
print('face_landmarks:', face_landmarks)
mp_drawing.draw_landmarks(
image=annotated_image,
landmark_list=face_landmarks,
connections=mp_face_mesh.FACE_CONNECTIONS,
landmark_drawing_spec=drawing_spec,
connection_drawing_spec=drawing_spec)
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', image)
face_mesh.close()
# For webcam input:
face_mesh = mp_face_mesh.FaceMesh(
min_detection_confidence=0.5, min_tracking_confidence=0.5)
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
cap = cv2.VideoCapture(0)
while cap.isOpened():
success, image = cap.read()
if not success:
break
# Flip the image horizontally for a later selfie-view display, and convert
# the BGR image to RGB.
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
# To improve performance, optionally mark the image as not writeable to
# pass by reference.
image.flags.writeable = False
results = face_mesh.process(image)
# Draw the face mesh annotations on the image.
image.flags.writeable = True
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
if results.multi_face_landmarks:
for face_landmarks in results.multi_face_landmarks:
mp_drawing.draw_landmarks(
image=image,
landmark_list=face_landmarks,
connections=mp_face_mesh.FACE_CONNECTIONS,
landmark_drawing_spec=drawing_spec,
connection_drawing_spec=drawing_spec)
cv2.imshow('MediaPipe FaceMesh', image)
if cv2.waitKey(5) & 0xFF == 27:
break
face_mesh.close()
cap.release()
Please refer to https://solutions.mediapipe.dev/face_mesh#python-solution-api
for usage examples.
"""
def __init__(self,
static_image_mode=False,
max_num_faces=2,
max_num_faces=1,
min_detection_confidence=0.5,
min_tracking_confidence=0.5):
"""Initializes a MediaPipe FaceMesh object.
Args:
static_image_mode: If set to False, the solution treats the input images
as a video stream. It will try to detect faces in the first input
images, and upon a successful detection further localizes the face
landmarks. In subsequent images, once all "max_num_faces" faces are
detected and the corresponding face landmarks are localized, it simply
tracks those landmarks without invoking another detection until it loses
track of any of the faces. This reduces latency and is ideal for
processing video frames. If set to True, face detection runs on every
input image, ideal for processing a batch of static, possibly unrelated,
images. Default to False.
max_num_faces: Maximum number of faces to detect. Default to 2.
min_detection_confidence: Minimum confidence value ([0.0, 1.0]) from the
face detection model for the detection to be considered successful.
Default to 0.5.
min_tracking_confidence: Minimum confidence value ([0.0, 1.0]) from the
landmark-tracking model for the face landmarks to be considered tracked
successfully, or otherwise face detection will be invoked automatically
on the next input image. Setting it to a higher value can increase
robustness of the solution, at the expense of a higher latency. Ignored
if "static_image_mode" is True, where face detection simply runs on
every image. Default to 0.5.
static_image_mode: Whether to treat the input images as a batch of static
and possibly unrelated images, or a video stream. See details in
https://solutions.mediapipe.dev/face_mesh#static-image-mode.
max_num_faces: Maximum number of faces to detect. See details in
https://solutions.mediapipe.dev/face_mesh#max-num-faces.
min_detection_confidence: Minimum confidence value ([0.0, 1.0]) for face
detection to be considered successful. See details in
https://solutions.mediapipe.dev/face_mesh#min-detection-confidence.
min_tracking_confidence: Minimum confidence value ([0.0, 1.0]) for the
face landmarks to be considered tracked successfully. See details in
https://solutions.mediapipe.dev/face_mesh#min-tracking-confidence.
"""
super().__init__(
binary_graph_path=BINARYPB_FILE_PATH,
side_inputs={
'num_faces': max_num_faces,
'can_skip_detection': not static_image_mode,
},
calculator_params={
'ConstantSidePacketCalculator.packet': [
constant_side_packet_calculator_pb2
.ConstantSidePacketCalculatorOptions.ConstantSidePacket(
bool_value=not static_image_mode)
],
'facedetectionfrontcpu__TensorsToDetectionsCalculator.min_score_thresh':
min_detection_confidence,
'facelandmarkcpu__ThresholdingCalculator.threshold':
mediapipe/python/solutions/face_mesh_test.py
+112
View File
@@ -0,0 +1,112 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for mediapipe.python.solutions.face_mesh."""
import os
from absl.testing import absltest
from absl.testing import parameterized
import cv2
import numpy as np
import numpy.testing as npt
# resources dependency
from mediapipe.python.solutions import face_mesh as mp_faces
TEST_IMAGE_PATH = 'mediapipe/python/solutions/testdata'
DIFF_THRESHOLOD = 20
EYE_INDICES_TO_LANDMARKS = {
33: [176, 350],
7: [177, 353],
163: [178, 357],
144: [179, 362],
145: [179, 369],
153: [179, 376],
154: [178, 382],
155: [177, 386],
133: [177, 388],
246: [175, 352],
161: [174, 355],
160: [172, 360],
159: [170, 367],
158: [171, 374],
157: [172, 381],
173: [175, 386],
263: [176, 475],
249: [177, 471],
390: [177, 467],
373: [178, 462],
374: [179, 454],
380: [179, 448],
381: [178, 441],
382: [177, 437],
362: [177, 435],
466: [175, 473],
388: [173, 469],
387: [171, 464],
386: [170, 457],
385: [171, 450],
384: [172, 443],
398: [175, 438]
}
class FaceMeshTest(parameterized.TestCase):
def test_invalid_image_shape(self):
faces = mp_faces.FaceMesh()
with self.assertRaisesRegex(
ValueError, 'Input image must contain three channel rgb data.'):
faces.process(np.arange(36, dtype=np.uint8).reshape(3, 3, 4))
def test_blank_image(self):
faces = mp_faces.FaceMesh()
image = np.zeros([100, 100, 3], dtype=np.uint8)
image.fill(255)
results = faces.process(image)
self.assertIsNone(results.multi_face_landmarks)
faces.close()
@parameterized.named_parameters(('static_image_mode', True, 1),
('video_mode', False, 5))
def test_face(self, static_image_mode: bool, num_frames: int):
image_path = os.path.join(os.path.dirname(__file__), 'testdata/face.jpg')
faces = mp_faces.FaceMesh(
static_image_mode=static_image_mode, min_detection_confidence=0.5)
image = cv2.flip(cv2.imread(image_path), 1)
def process_one_frame():
results = faces.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
multi_face_landmarks = []
for landmarks in results.multi_face_landmarks:
self.assertLen(landmarks.landmark, 468)
x = [landmark.x for landmark in landmarks.landmark]
y = [landmark.y for landmark in landmarks.landmark]
face_landmarks = np.transpose(np.stack((y, x))) * image.shape[0:2]
multi_face_landmarks.append(face_landmarks)
self.assertLen(multi_face_landmarks, 1)
# Verify the eye landmarks are correct as sanity check.
for idx, gt_lds in EYE_INDICES_TO_LANDMARKS.items():
prediction_error = np.abs(
np.asarray(multi_face_landmarks[0][idx]) - np.asarray(gt_lds))
npt.assert_array_less(prediction_error, DIFF_THRESHOLOD)
for _ in range(num_frames):
process_one_frame()
faces.close()
if __name__ == '__main__':
absltest.main()
mediapipe/python/solutions/hands.py
+20 -84
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""MediaPipe Hands."""
import enum
@@ -20,6 +19,7 @@ from typing import NamedTuple
import numpy as np
from mediapipe.calculators.core import constant_side_packet_calculator_pb2
# pylint: disable=unused-import
from mediapipe.calculators.core import gate_calculator_pb2
from mediapipe.calculators.core import split_vector_calculator_pb2
@@ -102,105 +102,41 @@ class Hands(SolutionBase):
horizontally. If that is not the case, use, for instance, cv2.flip(image, 1)
to flip the image first for a correct handedness output.
Usage examples:
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_hands = mp.solutions.hands
# For static images:
hands = mp_hands.Hands(
static_image_mode=True,
max_num_hands=2,
min_detection_confidence=0.7)
for idx, file in enumerate(file_list):
# Read an image, flip it around y-axis for correct handedness output (see
# above).
image = cv2.flip(cv2.imread(file), 1)
# Convert the BGR image to RGB before processing.
results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
# Print handedness and draw hand landmarks on the image.
print('handedness:', results.multi_handedness)
if not results.multi_hand_landmarks:
continue
annotated_image = image.copy()
for hand_landmarks in results.multi_hand_landmarks:
print('hand_landmarks:', hand_landmarks)
mp_drawing.draw_landmarks(
annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
cv2.imwrite(
'/tmp/annotated_image' + str(idx) + '.png', cv2.flip(image, 1))
hands.close()
# For webcam input:
hands = mp_hands.Hands(
min_detection_confidence=0.7, min_tracking_confidence=0.5)
cap = cv2.VideoCapture(0)
while cap.isOpened():
success, image = cap.read()
if not success:
break
# Flip the image horizontally for a later selfie-view display, and convert
# the BGR image to RGB.
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
# To improve performance, optionally mark the image as not writeable to
# pass by reference.
image.flags.writeable = False
results = hands.process(image)
# Draw the hand annotations on the image.
image.flags.writeable = True
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
mp_drawing.draw_landmarks(
image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
cv2.imshow('MediaPipe Hands', image)
if cv2.waitKey(5) & 0xFF == 27:
break
hands.close()
cap.release()
Please refer to https://solutions.mediapipe.dev/hands#python-solution-api for
usage examples.
"""
def __init__(self,
static_image_mode=False,
max_num_hands=2,
min_detection_confidence=0.7,
min_detection_confidence=0.5,
min_tracking_confidence=0.5):
"""Initializes a MediaPipe Hand object.
Args:
static_image_mode: If set to False, the solution treats the input images
as a video stream. It will try to detect hands in the first input
images, and upon a successful detection further localizes the hand
landmarks. In subsequent images, once all "max_num_hands" hands are
detected and the corresponding hand landmarks are localized, it simply
tracks those landmarks without invoking another detection until it loses
track of any of the hands. This reduces latency and is ideal for
processing video frames. If set to True, hand detection runs on every
input image, ideal for processing a batch of static, possibly unrelated,
images. Default to False.
max_num_hands: Maximum number of hands to detect. Default to 2.
min_detection_confidence: Minimum confidence value ([0.0, 1.0]) from the
hand detection model for the detection to be considered successful.
Default to 0.7.
min_tracking_confidence: Minimum confidence value ([0.0, 1.0]) from the
landmark-tracking model for the hand landmarks to be considered tracked
successfully, or otherwise hand detection will be invoked automatically
on the next input image. Setting it to a higher value can increase
robustness of the solution, at the expense of a higher latency. Ignored
if "static_image_mode" is True, where hand detection simply runs on
every image. Default to 0.5.
static_image_mode: Whether to treat the input images as a batch of static
and possibly unrelated images, or a video stream. See details in
https://solutions.mediapipe.dev/hands#static-image-mode.
max_num_hands: Maximum number of hands to detect. See details in
https://solutions.mediapipe.dev/hands#max-num-hands.
min_detection_confidence: Minimum confidence value ([0.0, 1.0]) for hand
detection to be considered successful. See details in
https://solutions.mediapipe.dev/hands#min-detection-confidence.
min_tracking_confidence: Minimum confidence value ([0.0, 1.0]) for the
hand landmarks to be considered tracked successfully. See details in
https://solutions.mediapipe.dev/hands#min-tracking-confidence.
"""
super().__init__(
binary_graph_path=BINARYPB_FILE_PATH,
side_inputs={
'num_hands': max_num_hands,
'can_skip_detection': not static_image_mode,
},
calculator_params={
'ConstantSidePacketCalculator.packet': [
constant_side_packet_calculator_pb2
.ConstantSidePacketCalculatorOptions.ConstantSidePacket(
bool_value=not static_image_mode)
],
'palmdetectioncpu__TensorsToDetectionsCalculator.min_score_thresh':
min_detection_confidence,
'handlandmarkcpu__ThresholdingCalculator.threshold':
mediapipe/python/solutions/hands_test.py
+99
View File
@@ -0,0 +1,99 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for mediapipe.python.solutions.hands."""
import os
from absl.testing import absltest
from absl.testing import parameterized
import cv2
import numpy as np
import numpy.testing as npt
# resources dependency
from mediapipe.python.solutions import hands as mp_hands
TEST_IMAGE_PATH = 'mediapipe/python/solutions/testdata'
DIFF_THRESHOLOD = 20
EXPECTED_HAND_COORDINATES_PREDICTION = [[[332, 144], [323, 211], [286, 257],
[237, 289], [203, 322], [216, 219],
[138, 238], [90, 249], [51, 253],
[204, 177], [115, 184], [60, 187],
[19, 185], [208, 138], [127, 131],
[77, 124], [36, 117], [222, 106],
[159, 92], [124, 79], [93, 68]],
[[43, 570], [56, 504], [94, 459],
[146, 429], [182, 397], [167, 496],
[245, 479], [292, 469], [330, 464],
[177, 540], [265, 534], [319, 533],
[360, 536], [172, 581], [252, 587],
[304, 593], [346, 599], [157, 615],
[219, 628], [255, 638], [288, 648]]]
class HandsTest(parameterized.TestCase):
def test_invalid_image_shape(self):
hands = mp_hands.Hands()
with self.assertRaisesRegex(
ValueError, 'Input image must contain three channel rgb data.'):
hands.process(np.arange(36, dtype=np.uint8).reshape(3, 3, 4))
def test_blank_image(self):
hands = mp_hands.Hands()
image = np.zeros([100, 100, 3], dtype=np.uint8)
image.fill(255)
results = hands.process(image)
self.assertIsNone(results.multi_hand_landmarks)
self.assertIsNone(results.multi_handedness)
hands.close()
@parameterized.named_parameters(('static_image_mode', True, 1),
('video_mode', False, 5))
def test_multi_hands(self, static_image_mode, num_frames):
image_path = os.path.join(os.path.dirname(__file__), 'testdata/hands.jpg')
hands = mp_hands.Hands(
static_image_mode=static_image_mode,
max_num_hands=2,
min_detection_confidence=0.5)
image = cv2.flip(cv2.imread(image_path), 1)
def process_one_frame():
results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
handedness = [
handedness.classification[0].label
for handedness in results.multi_handedness
]
self.assertLen(handedness, 2)
multi_hand_coordinates = []
for landmarks in results.multi_hand_landmarks:
self.assertLen(landmarks.landmark, 21)
x = [landmark.x for landmark in landmarks.landmark]
y = [landmark.y for landmark in landmarks.landmark]
hand_coordinates = np.transpose(np.stack((y, x))) * image.shape[0:2]
multi_hand_coordinates.append(hand_coordinates)
self.assertLen(multi_hand_coordinates, 2)
prediction_error = np.abs(
np.asarray(multi_hand_coordinates) -
np.asarray(EXPECTED_HAND_COORDINATES_PREDICTION))
npt.assert_array_less(prediction_error, DIFF_THRESHOLOD)
for _ in range(num_frames):
process_one_frame()
hands.close()
if __name__ == '__main__':
absltest.main()
mediapipe/python/solutions/holistic.py
+130
View File
@@ -0,0 +1,130 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe Holistic."""
from typing import NamedTuple
import numpy as np
from mediapipe.calculators.core import constant_side_packet_calculator_pb2
# pylint: disable=unused-import
from mediapipe.calculators.core import gate_calculator_pb2
from mediapipe.calculators.core import split_vector_calculator_pb2
from mediapipe.calculators.tensor import image_to_tensor_calculator_pb2
from mediapipe.calculators.tensor import inference_calculator_pb2
from mediapipe.calculators.tensor import tensors_to_classification_calculator_pb2
from mediapipe.calculators.tensor import tensors_to_floats_calculator_pb2
from mediapipe.calculators.tensor import tensors_to_landmarks_calculator_pb2
from mediapipe.calculators.tflite import ssd_anchors_calculator_pb2
from mediapipe.calculators.util import detections_to_rects_calculator_pb2
from mediapipe.calculators.util import landmark_projection_calculator_pb2
from mediapipe.calculators.util import non_max_suppression_calculator_pb2
from mediapipe.calculators.util import rect_transformation_calculator_pb2
from mediapipe.modules.holistic_landmark.calculators import roi_tracking_calculator_pb2
# pylint: enable=unused-import
from mediapipe.python.solution_base import SolutionBase
# pylint: disable=unused-import
from mediapipe.python.solutions.face_mesh import FACE_CONNECTIONS
from mediapipe.python.solutions.hands import HAND_CONNECTIONS
from mediapipe.python.solutions.hands import HandLandmark
from mediapipe.python.solutions.pose import POSE_CONNECTIONS
from mediapipe.python.solutions.pose import PoseLandmark
# pylint: enable=unused-import
BINARYPB_FILE_PATH = 'mediapipe/modules/holistic_landmark/holistic_landmark_cpu.binarypb'
class Holistic(SolutionBase):
"""MediaPipe Holistic.
MediaPipe Holistic processes an RGB image and returns pose landmarks, left and
right hand landmarks, and face mesh landmarks on the most prominent person
detected.
Please refer to https://solutions.mediapipe.dev/holistic#python-solution-api
for usage examples.
"""
def __init__(self,
static_image_mode=False,
upper_body_only=False,
smooth_landmarks=True,
min_detection_confidence=0.5,
min_tracking_confidence=0.5):
"""Initializes a MediaPipe Holistic object.
Args:
static_image_mode: Whether to treat the input images as a batch of static
and possibly unrelated images, or a video stream. See details in
https://solutions.mediapipe.dev/holistic#static-image-mode.
upper_body_only: Whether to track the full set of 33 pose landmarks or
only the 25 upper-body pose landmarks. See details in
https://solutions.mediapipe.dev/holistic#upper-body-only.
smooth_landmarks: Whether to filter landmarks across different input
images to reduce jitter. See details in
https://solutions.mediapipe.dev/holistic#smooth_landmarks.
min_detection_confidence: Minimum confidence value ([0.0, 1.0]) for person
detection to be considered successful. See details in
https://solutions.mediapipe.dev/holistic#min-detection-confidence.
min_tracking_confidence: Minimum confidence value ([0.0, 1.0]) for the
pose landmarks to be considered tracked successfully. See details in
https://solutions.mediapipe.dev/holistic#min-tracking-confidence.
"""
super().__init__(
binary_graph_path=BINARYPB_FILE_PATH,
side_inputs={
'upper_body_only': upper_body_only,
'smooth_landmarks': smooth_landmarks and not static_image_mode,
},
calculator_params={
'poselandmarkcpu__ConstantSidePacketCalculator.packet': [
constant_side_packet_calculator_pb2
.ConstantSidePacketCalculatorOptions.ConstantSidePacket(
bool_value=not static_image_mode)
],
'poselandmarkcpu__posedetectioncpu__TensorsToDetectionsCalculator.min_score_thresh':
min_detection_confidence,
'poselandmarkcpu__poselandmarkbyroicpu__ThresholdingCalculator.threshold':
min_tracking_confidence,
},
outputs=[
'pose_landmarks', 'left_hand_landmarks', 'right_hand_landmarks',
'face_landmarks'
])
def process(self, image: np.ndarray) -> NamedTuple:
"""Processes an RGB image and returns the pose landmarks, left and right hand landmarks, and face landmarks on the most prominent person detected.
Args:
image: An RGB image represented as a numpy ndarray.
Raises:
RuntimeError: If the underlying graph occurs any error.
ValueError: If the input image is not three channel RGB.
Returns:
A NamedTuple that has four fields:
1) "pose_landmarks" field that contains the pose landmarks on the most
prominent person detected.
2) "left_hand_landmarks" and "right_hand_landmarks" fields that contain
the left and right hand landmarks of the most prominent person detected.
3) "face_landmarks" field that contains the face landmarks of the most
prominent person detected.
"""
results = super().process(input_data={'image': image})
if results.pose_landmarks:
for landmark in results.pose_landmarks.landmark:
landmark.ClearField('presence')
return results
mediapipe/python/solutions/holistic_test.py
+143
View File
@@ -0,0 +1,143 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for mediapipe.python.solutions.pose."""
import math
import os
from absl.testing import absltest
from absl.testing import parameterized
import cv2
import numpy as np
import numpy.testing as npt
# resources dependency
from mediapipe.python.solutions import holistic as mp_holistic
TEST_IMAGE_PATH = 'mediapipe/python/solutions/testdata'
POSE_DIFF_THRESHOLOD = 30 # pixels
HAND_DIFF_THRESHOLOD = 10 # pixels
EXPECTED_POSE_COORDINATES_PREDICTION = [[593, 645], [593, 626], [599, 621],
[605, 617], [575, 637], [569, 640],
[563, 643], [621, 616], [565, 652],
[617, 652], [595, 667], [714, 662],
[567, 749], [792, 559], [497, 844],
[844, 435], [407, 906], [866, 403],
[381, 921], [859, 392], [366, 922],
[850, 405], [381, 918], [707, 948],
[631, 940], [582, 1122], [599, 1097],
[495, 1277], [641, 1239], [485, 1300],
[658, 1257], [453, 1332], [626, 1308]]
EXPECTED_LEFT_HAND_COORDINATES_PREDICTION = [[843, 404], [862, 395], [876, 383],
[887, 369], [896, 359], [854, 367],
[868, 347], [879, 346], [885, 349],
[843, 362], [859, 341], [871, 340],
[878, 344], [837, 361], [849, 341],
[859, 338], [867, 339], [834, 361],
[841, 346], [848, 342], [854, 341]]
EXPECTED_RIGHT_HAND_COORDINATES_PREDICTION = [[391, 934], [371,
930], [354, 930],
[340, 934], [328,
939], [350, 938],
[339, 946], [347,
951], [355, 952],
[356, 946], [346,
955], [358, 956],
[366, 953], [361,
952], [354, 959],
[364, 958], [372,
954], [366, 957],
[359, 963], [364, 962],
[368, 960]]
class PoseTest(parameterized.TestCase):
def _verify_output_landmarks(self, landmark_list, image_shape, num_landmarks,
expected_results, diff_thresholds):
self.assertLen(landmark_list.landmark, num_landmarks)
image_rows, image_cols, _ = image_shape
pose_coordinates = [(math.floor(landmark.x * image_cols),
math.floor(landmark.y * image_rows))
for landmark in landmark_list.landmark]
prediction_error = np.abs(
np.asarray(pose_coordinates) -
np.asarray(expected_results[:num_landmarks]))
npt.assert_array_less(prediction_error, diff_thresholds)
def test_invalid_image_shape(self):
holistic = mp_holistic.Holistic()
with self.assertRaisesRegex(
ValueError, 'Input image must contain three channel rgb data.'):
holistic.process(np.arange(36, dtype=np.uint8).reshape(3, 3, 4))
def test_blank_image(self):
holistic = mp_holistic.Holistic()
image = np.zeros([100, 100, 3], dtype=np.uint8)
image.fill(255)
results = holistic.process(image)
self.assertIsNone(results.pose_landmarks)
holistic.close()
@parameterized.named_parameters(('static_image_mode', True, 3),
('video_mode', False, 3))
def test_upper_body_model(self, static_image_mode, num_frames):
image_path = os.path.join(os.path.dirname(__file__), 'testdata/pose.jpg')
holistic = mp_holistic.Holistic(
static_image_mode=static_image_mode, upper_body_only=True)
image = cv2.imread(image_path)
for _ in range(num_frames):
results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
self._verify_output_landmarks(results.pose_landmarks, image.shape, 25,
EXPECTED_POSE_COORDINATES_PREDICTION,
POSE_DIFF_THRESHOLOD)
self._verify_output_landmarks(results.left_hand_landmarks, image.shape,
21,
EXPECTED_LEFT_HAND_COORDINATES_PREDICTION,
HAND_DIFF_THRESHOLOD)
self._verify_output_landmarks(results.right_hand_landmarks, image.shape,
21,
EXPECTED_RIGHT_HAND_COORDINATES_PREDICTION,
HAND_DIFF_THRESHOLOD)
# TODO: Verify the correctness of the face landmarks.
self.assertLen(results.face_landmarks.landmark, 468)
holistic.close()
@parameterized.named_parameters(('static_image_mode', True, 3),
('video_mode', False, 3))
def test_full_body_model(self, static_image_mode, num_frames):
image_path = os.path.join(os.path.dirname(__file__), 'testdata/pose.jpg')
holistic = mp_holistic.Holistic(static_image_mode=static_image_mode)
image = cv2.imread(image_path)
for _ in range(num_frames):
results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
self._verify_output_landmarks(results.pose_landmarks, image.shape, 33,
EXPECTED_POSE_COORDINATES_PREDICTION,
POSE_DIFF_THRESHOLOD)
self._verify_output_landmarks(results.left_hand_landmarks, image.shape,
21,
EXPECTED_LEFT_HAND_COORDINATES_PREDICTION,
HAND_DIFF_THRESHOLOD)
self._verify_output_landmarks(results.right_hand_landmarks, image.shape,
21,
EXPECTED_RIGHT_HAND_COORDINATES_PREDICTION,
HAND_DIFF_THRESHOLOD)
# TODO: Verify the correctness of the face landmarks.
self.assertLen(results.face_landmarks.landmark, 468)
holistic.close()
if __name__ == '__main__':
absltest.main()
mediapipe/python/solutions/pose.py
+80 -101
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""MediaPipe Pose."""
import enum
@@ -20,6 +19,7 @@ from typing import NamedTuple
import numpy as np
from mediapipe.calculators.core import constant_side_packet_calculator_pb2
# pylint: disable=unused-import
from mediapipe.calculators.core import gate_calculator_pb2
from mediapipe.calculators.core import split_vector_calculator_pb2
@@ -28,6 +28,7 @@ from mediapipe.calculators.tensor import inference_calculator_pb2
from mediapipe.calculators.tensor import tensors_to_classification_calculator_pb2
from mediapipe.calculators.tensor import tensors_to_detections_calculator_pb2
from mediapipe.calculators.tensor import tensors_to_landmarks_calculator_pb2
from mediapipe.calculators.tflite import ssd_anchors_calculator_pb2
from mediapipe.calculators.util import detections_to_rects_calculator_pb2
from mediapipe.calculators.util import landmarks_smoothing_calculator_pb2
from mediapipe.calculators.util import logic_calculator_pb2
@@ -41,33 +42,40 @@ from mediapipe.python.solution_base import SolutionBase
class PoseLandmark(enum.IntEnum):
"""The 25 (upper-body) pose landmarks."""
NOSE = 0
RIGHT_EYE_INNER = 1
RIGHT_EYE = 2
RIGHT_EYE_OUTER = 3
LEFT_EYE_INNER = 4
LEFT_EYE = 5
LEFT_EYE_OUTER = 6
RIGHT_EAR = 7
LEFT_EAR = 8
MOUTH_RIGHT = 9
MOUTH_LEFT = 10
RIGHT_SHOULDER = 11
LEFT_SHOULDER = 12
RIGHT_ELBOW = 13
LEFT_ELBOW = 14
RIGHT_WRIST = 15
LEFT_WRIST = 16
RIGHT_PINKY = 17
LEFT_PINKY = 18
RIGHT_INDEX = 19
LEFT_INDEX = 20
RIGHT_THUMB = 21
LEFT_THUMB = 22
RIGHT_HIP = 23
LEFT_HIP = 24
LEFT_EYE_INNER = 1
LEFT_EYE = 2
LEFT_EYE_OUTER = 3
RIGHT_EYE_INNER = 4
RIGHT_EYE = 5
RIGHT_EYE_OUTER = 6
LEFT_EAR = 7
RIGHT_EAR = 8
MOUTH_LEFT = 9
MOUTH_RIGHT = 10
LEFT_SHOULDER = 11
RIGHT_SHOULDER = 12
LEFT_ELBOW = 13
RIGHT_ELBOW = 14
LEFT_WRIST = 15
RIGHT_WRIST = 16
LEFT_PINKY = 17
RIGHT_PINKY = 18
LEFT_INDEX = 19
RIGHT_INDEX = 20
LEFT_THUMB = 21
RIGHT_THUMB = 22
LEFT_HIP = 23
RIGHT_HIP = 24
LEFT_KNEE = 25
RIGHT_KNEE = 26
LEFT_ANKLE = 27
RIGHT_ANKLE = 28
LEFT_HEEL = 29
RIGHT_HEEL = 30
LEFT_FOOT_INDEX = 31
RIGHT_FOOT_INDEX = 32
BINARYPB_FILE_PATH = 'mediapipe/modules/pose_landmark/pose_landmark_upper_body_smoothed_cpu.binarypb'
BINARYPB_FILE_PATH = 'mediapipe/modules/pose_landmark/pose_landmark_cpu.binarypb'
POSE_CONNECTIONS = frozenset([
(PoseLandmark.NOSE, PoseLandmark.RIGHT_EYE_INNER),
(PoseLandmark.RIGHT_EYE_INNER, PoseLandmark.RIGHT_EYE),
@@ -93,7 +101,18 @@ POSE_CONNECTIONS = frozenset([
(PoseLandmark.LEFT_PINKY, PoseLandmark.LEFT_INDEX),
(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP),
(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP),
(PoseLandmark.RIGHT_HIP, PoseLandmark.LEFT_HIP)
(PoseLandmark.RIGHT_HIP, PoseLandmark.LEFT_HIP),
(PoseLandmark.RIGHT_HIP, PoseLandmark.LEFT_HIP),
(PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE),
(PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE),
(PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE),
(PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE),
(PoseLandmark.RIGHT_ANKLE, PoseLandmark.RIGHT_HEEL),
(PoseLandmark.LEFT_ANKLE, PoseLandmark.LEFT_HEEL),
(PoseLandmark.RIGHT_HEEL, PoseLandmark.RIGHT_FOOT_INDEX),
(PoseLandmark.LEFT_HEEL, PoseLandmark.LEFT_FOOT_INDEX),
(PoseLandmark.RIGHT_ANKLE, PoseLandmark.RIGHT_FOOT_INDEX),
(PoseLandmark.LEFT_ANKLE, PoseLandmark.LEFT_FOOT_INDEX),
])
@@ -103,94 +122,50 @@ class Pose(SolutionBase):
MediaPipe Pose processes an RGB image and returns pose landmarks on the most
prominent person detected.
Usage examples:
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_pose = mp.solutions.pose
# For static images:
pose = mp_pose.Pose(
static_image_mode=True, min_detection_confidence=0.5)
for idx, file in enumerate(file_list):
image = cv2.imread(file)
# Convert the BGR image to RGB before processing.
results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
# Print and draw pose landmarks on the image.
print(
'nose landmark:',
results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE])
annotated_image = image.copy()
mp_drawing.draw_landmarks(
annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', image)
pose.close()
# For webcam input:
pose = mp_pose.Pose(
min_detection_confidence=0.5, min_tracking_confidence=0.5)
cap = cv2.VideoCapture(0)
while cap.isOpened():
success, image = cap.read()
if not success:
break
# Flip the image horizontally for a later selfie-view display, and convert
# the BGR image to RGB.
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
# To improve performance, optionally mark the image as not writeable to
# pass by reference.
image.flags.writeable = False
results = pose.process(image)
# Draw the pose annotation on the image.
image.flags.writeable = True
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
mp_drawing.draw_landmarks(
image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
cv2.imshow('MediaPipe Pose', image)
if cv2.waitKey(5) & 0xFF == 27:
break
pose.close()
cap.release()
Please refer to https://solutions.mediapipe.dev/pose#python-solution-api for
usage examples.
"""
def __init__(self,
static_image_mode=False,
upper_body_only=False,
smooth_landmarks=True,
min_detection_confidence=0.5,
min_tracking_confidence=0.5):
"""Initializes a MediaPipe Pose object.
Args:
static_image_mode: If set to False, the solution treats the input images
as a video stream. It will try to detect the most prominent person in
the very first images, and upon a successful detection further localizes
the pose landmarks. In subsequent images, it then simply tracks those
landmarks without invoking another detection until it loses track, on
reducing computation and latency. If set to True, person detection runs
every input image, ideal for processing a batch of static, possibly
unrelated, images. Default to False.
min_detection_confidence: Minimum confidence value ([0.0, 1.0]) from the
person-detection model for the detection to be considered successful.
Default to 0.5.
min_tracking_confidence: Minimum confidence value ([0.0, 1.0]) from the
landmark-tracking model for the pose landmarks to be considered tracked
successfully, or otherwise person detection will be invoked
automatically on the next input image. Setting it to a higher value can
increase robustness of the solution, at the expense of a higher latency.
Ignored if "static_image_mode" is True, where person detection simply
runs on every image. Default to 0.5.
static_image_mode: Whether to treat the input images as a batch of static
and possibly unrelated images, or a video stream. See details in
https://solutions.mediapipe.dev/pose#static-image-mode.
upper_body_only: Whether to track the full set of 33 pose landmarks or
only the 25 upper-body pose landmarks. See details in
https://solutions.mediapipe.dev/pose#upper-body-only.
smooth_landmarks: Whether to filter landmarks across different input
images to reduce jitter. See details in
https://solutions.mediapipe.dev/pose#smooth_landmarks.
min_detection_confidence: Minimum confidence value ([0.0, 1.0]) for person
detection to be considered successful. See details in
https://solutions.mediapipe.dev/pose#min-detection-confidence.
min_tracking_confidence: Minimum confidence value ([0.0, 1.0]) for the
pose landmarks to be considered tracked successfully. See details in
https://solutions.mediapipe.dev/pose#min-tracking-confidence.
"""
super().__init__(
binary_graph_path=BINARYPB_FILE_PATH,
side_inputs={
'can_skip_detection': not static_image_mode,
'upper_body_only': upper_body_only,
'smooth_landmarks': smooth_landmarks and not static_image_mode,
},
calculator_params={
'poselandmarkupperbodycpu__posedetectioncpu__TensorsToDetectionsCalculator.min_score_thresh':
'ConstantSidePacketCalculator.packet': [
constant_side_packet_calculator_pb2
.ConstantSidePacketCalculatorOptions.ConstantSidePacket(
bool_value=not static_image_mode)
],
'poselandmarkcpu__posedetectioncpu__TensorsToDetectionsCalculator.min_score_thresh':
min_detection_confidence,
'poselandmarkupperbodycpu__poselandmarkupperbodybyroicpu__ThresholdingCalculator.threshold':
'poselandmarkcpu__poselandmarkbyroicpu__ThresholdingCalculator.threshold':
min_tracking_confidence,
},
outputs=['pose_landmarks'])
@@ -210,4 +185,8 @@ class Pose(SolutionBase):
landmarks on the most prominent person detected.
"""
return super().process(input_data={'image': image})
results = super().process(input_data={'image': image})
if results.pose_landmarks:
for landmark in results.pose_landmarks.landmark:
landmark.ClearField('presence')
return results
mediapipe/python/solutions/pose_test.py
+97
View File
@@ -0,0 +1,97 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for mediapipe.python.solutions.pose."""
import math
import os
from absl.testing import absltest
from absl.testing import parameterized
import cv2
import numpy as np
import numpy.testing as npt
# resources dependency
from mediapipe.python.solutions import pose as mp_pose
TEST_IMAGE_PATH = 'mediapipe/python/solutions/testdata'
DIFF_THRESHOLOD = 30 # pixels
EXPECTED_POSE_COORDINATES_PREDICTION = [[593, 645], [593, 626], [599, 621],
[605, 617], [575, 637], [569, 640],
[563, 643], [621, 616], [565, 652],
[617, 652], [595, 667], [714, 662],
[567, 749], [792, 559], [497, 844],
[844, 435], [407, 906], [866, 403],
[381, 921], [859, 392], [366, 922],
[850, 405], [381, 918], [707, 948],
[631, 940], [582, 1122], [599, 1097],
[495, 1277], [641, 1239], [485, 1300],
[658, 1257], [453, 1332], [626, 1308]]
class PoseTest(parameterized.TestCase):
def _verify_output_landmarks(self, landmark_list, image_shape, num_landmarks):
self.assertLen(landmark_list.landmark, num_landmarks)
image_rows, image_cols, _ = image_shape
pose_coordinates = [(math.floor(landmark.x * image_cols),
math.floor(landmark.y * image_rows))
for landmark in landmark_list.landmark]
prediction_error = np.abs(
np.asarray(pose_coordinates) -
np.asarray(EXPECTED_POSE_COORDINATES_PREDICTION[:num_landmarks]))
npt.assert_array_less(prediction_error, DIFF_THRESHOLOD)
def test_invalid_image_shape(self):
pose = mp_pose.Pose()
with self.assertRaisesRegex(
ValueError, 'Input image must contain three channel rgb data.'):
pose.process(np.arange(36, dtype=np.uint8).reshape(3, 3, 4))
def test_blank_image(self):
pose = mp_pose.Pose()
image = np.zeros([100, 100, 3], dtype=np.uint8)
image.fill(255)
results = pose.process(image)
self.assertIsNone(results.pose_landmarks)
pose.close()
@parameterized.named_parameters(('static_image_mode', True, 3),
('video_mode', False, 3))
def test_upper_body_model(self, static_image_mode, num_frames):
image_path = os.path.join(os.path.dirname(__file__), 'testdata/pose.jpg')
pose = mp_pose.Pose(static_image_mode=static_image_mode,
upper_body_only=True)
image = cv2.imread(image_path)
for _ in range(num_frames):
results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
self._verify_output_landmarks(results.pose_landmarks, image.shape, 25)
pose.close()
@parameterized.named_parameters(('static_image_mode', True, 3),
('video_mode', False, 3))
def test_full_body_model(self, static_image_mode, num_frames):
image_path = os.path.join(os.path.dirname(__file__), 'testdata/pose.jpg')
pose = mp_pose.Pose(static_image_mode=static_image_mode)
image = cv2.imread(image_path)
for _ in range(num_frames):
results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
self._verify_output_landmarks(results.pose_landmarks, image.shape, 33)
pose.close()
if __name__ == '__main__':
absltest.main()
mediapipe/python/timestamp_test.py
+6 -6
View File
@@ -22,25 +22,25 @@ import mediapipe as mp
class TimestampTest(absltest.TestCase):
def testTimesatmp(self):
def test_timestamp(self):
t = mp.Timestamp(100)
self.assertEqual(t.value, 100)
self.assertEqual(t, 100)
self.assertEqual(str(t), '<mediapipe.Timestamp with value: 100>')
def testTimestampCopyConstructor(self):
def test_timestamp_copy_constructor(self):
ts1 = mp.Timestamp(100)
ts2 = mp.Timestamp(ts1)
self.assertEqual(ts1, ts2)
def testTimesatmpComparsion(self):
def test_timestamp_comparsion(self):
ts1 = mp.Timestamp(100)
ts2 = mp.Timestamp(100)
self.assertEqual(ts1, ts2)
ts3 = mp.Timestamp(200)
self.assertNotEqual(ts1, ts3)
def testTimesatmpSpecialValues(self):
def test_timestamp_special_values(self):
t1 = mp.Timestamp.UNSET
self.assertEqual(str(t1), '<mediapipe.Timestamp with value: UNSET>')
t2 = mp.Timestamp.UNSTARTED
@@ -56,7 +56,7 @@ class TimestampTest(absltest.TestCase):
t7 = mp.Timestamp.DONE
self.assertEqual(str(t7), '<mediapipe.Timestamp with value: DONE>')
def testTimestampComparisons(self):
def test_timestamp_comparisons(self):
ts1 = mp.Timestamp(100)
ts2 = mp.Timestamp(101)
self.assertGreater(ts2, ts1)
@@ -65,7 +65,7 @@ class TimestampTest(absltest.TestCase):
self.assertLessEqual(ts1, ts2)
self.assertNotEqual(ts1, ts2)
def testFromSeconds(self):
def test_from_seconds(self):
now = time.time()
ts = mp.Timestamp.from_seconds(now)
self.assertAlmostEqual(now, ts.seconds(), delta=1)