Project import generated by Copybara.

GitOrigin-RevId: 9295f8ea2339edb71073695ed4fb3fded2f48c60

mediapipe/python/BUILD

@@ -0,0 +1,50 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@pybind11_bazel//:build_defs.bzl", "pybind_extension")
+
+licenses(["notice"])  # Apache 2.0
+
+cc_library(
+    name = "builtin_calculators",
+    deps = [
+        "//mediapipe/calculators/core:pass_through_calculator",
+        "//mediapipe/graphs/pose_tracking:upper_body_pose_tracking_cpu_deps",
+    ],
+)
+
+pybind_extension(
+    name = "_framework_bindings",
+    srcs = ["framework_bindings.cc"],
+    linkopts = [
+        "-lopencv_core",
+        "-lopencv_imgproc",
+        "-lopencv_highgui",
+        "-lopencv_video",
+        "-lopencv_features2d",
+        "-lopencv_calib3d",
+        "-lopencv_imgcodecs",
+    ],
+    deps = [
+        ":builtin_calculators",
+        "//mediapipe/python/pybind:calculator_graph",
+        "//mediapipe/python/pybind:image_frame",
+        "//mediapipe/python/pybind:matrix",
+        "//mediapipe/python/pybind:packet",
+        "//mediapipe/python/pybind:packet_creator",
+        "//mediapipe/python/pybind:packet_getter",
+        "//mediapipe/python/pybind:resource_util",
+        "//mediapipe/python/pybind:timestamp",
+    ],
+)

mediapipe/python/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""MediaPipe Python API."""
+
+from mediapipe.python._framework_bindings import resource_util
+from mediapipe.python._framework_bindings.calculator_graph import CalculatorGraph
+from mediapipe.python._framework_bindings.calculator_graph import GraphInputStreamAddMode
+from mediapipe.python._framework_bindings.image_frame import ImageFormat
+from mediapipe.python._framework_bindings.image_frame import ImageFrame
+from mediapipe.python._framework_bindings.matrix import Matrix
+from mediapipe.python._framework_bindings.packet import Packet
+from mediapipe.python._framework_bindings.timestamp import Timestamp
+import mediapipe.python.packet_creator
+import mediapipe.python.packet_getter

mediapipe/python/calculator_graph_test.py

@@ -0,0 +1,178 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Tests for mediapipe.python._framework_bindings.calculator_graph."""
+
+# Dependency imports
+
+from absl.testing import absltest
+import mediapipe.python as mp
+from google.protobuf import text_format
+from mediapipe.framework import calculator_pb2
+
+
+class GraphTest(absltest.TestCase):
+
+  def testInvalidBinaryGraphFile(self):
+    with self.assertRaisesRegex(FileNotFoundError, 'No such file or directory'):
+      mp.CalculatorGraph(binary_graph_path='/tmp/abc.binarypb')
+
+  def testInvalidNodeConfig(self):
+    text_config = """
+      node {
+        calculator: 'PassThroughCalculator'
+        input_stream: 'in'
+        input_stream: 'in'
+        output_stream: 'out'
+      }
+    """
+    config_proto = calculator_pb2.CalculatorGraphConfig()
+    text_format.Parse(text_config, config_proto)
+    with self.assertRaisesRegex(
+        ValueError,
+        'Input and output streams to PassThroughCalculator must use matching tags and indexes.'
+    ):
+      mp.CalculatorGraph(graph_config=config_proto)
+
+  def testInvalidCalculatorType(self):
+    text_config = """
+      node {
+        calculator: 'SomeUnknownCalculator'
+        input_stream: 'in'
+        output_stream: 'out'
+      }
+    """
+    config_proto = calculator_pb2.CalculatorGraphConfig()
+    text_format.Parse(text_config, config_proto)
+    with self.assertRaisesRegex(
+        RuntimeError, 'Unable to find Calculator \"SomeUnknownCalculator\"'):
+      mp.CalculatorGraph(graph_config=config_proto)
+
+  def testGraphInitializedWithProtoConfig(self):
+    text_config = """
+      max_queue_size: 1
+      input_stream: 'in'
+      output_stream: 'out'
+      node {
+        calculator: 'PassThroughCalculator'
+        input_stream: 'in'
+        output_stream: 'out'
+      }
+    """
+    config_proto = calculator_pb2.CalculatorGraphConfig()
+    text_format.Parse(text_config, config_proto)
+    graph = mp.CalculatorGraph(graph_config=config_proto)
+
+    hello_world_packet = mp.packet_creator.create_string('hello world')
+    out = []
+    graph = mp.CalculatorGraph(graph_config=config_proto)
+    graph.observe_output_stream('out', lambda _, packet: out.append(packet))
+    graph.start_run()
+    graph.add_packet_to_input_stream(
+        stream='in', packet=hello_world_packet, timestamp=0)
+    graph.add_packet_to_input_stream(
+        stream='in', packet=hello_world_packet.at(1))
+    graph.close()
+    self.assertEqual(graph.graph_input_stream_add_mode,
+                     mp.GraphInputStreamAddMode.WAIT_TILL_NOT_FULL)
+    self.assertEqual(graph.max_queue_size, 1)
+    self.assertFalse(graph.has_error())
+    self.assertLen(out, 2)
+    self.assertEqual(out[0].timestamp, 0)
+    self.assertEqual(out[1].timestamp, 1)
+    self.assertEqual(mp.packet_getter.get_str(out[0]), 'hello world')
+    self.assertEqual(mp.packet_getter.get_str(out[1]), 'hello world')
+
+  def testGraphInitializedWithTextConfig(self):
+    text_config = """
+      max_queue_size: 1
+      input_stream: 'in'
+      output_stream: 'out'
+      node {
+        calculator: 'PassThroughCalculator'
+        input_stream: 'in'
+        output_stream: 'out'
+      }
+    """
+
+    hello_world_packet = mp.packet_creator.create_string('hello world')
+    out = []
+    graph = mp.CalculatorGraph(graph_config=text_config)
+    graph.observe_output_stream('out', lambda _, packet: out.append(packet))
+    graph.start_run()
+    graph.add_packet_to_input_stream(
+        stream='in', packet=hello_world_packet.at(0))
+    graph.add_packet_to_input_stream(
+        stream='in', packet=hello_world_packet, timestamp=1)
+    graph.close()
+    self.assertEqual(graph.graph_input_stream_add_mode,
+                     mp.GraphInputStreamAddMode.WAIT_TILL_NOT_FULL)
+    self.assertEqual(graph.max_queue_size, 1)
+    self.assertFalse(graph.has_error())
+    self.assertLen(out, 2)
+    self.assertEqual(out[0].timestamp, 0)
+    self.assertEqual(out[1].timestamp, 1)
+    self.assertEqual(mp.packet_getter.get_str(out[0]), 'hello world')
+    self.assertEqual(mp.packet_getter.get_str(out[1]), 'hello world')
+
+  def testInsertPacketsWithSameTimestamp(self):
+    text_config = """
+      max_queue_size: 1
+      input_stream: 'in'
+      output_stream: 'out'
+      node {
+        calculator: 'PassThroughCalculator'
+        input_stream: 'in'
+        output_stream: 'out'
+      }
+    """
+    config_proto = calculator_pb2.CalculatorGraphConfig()
+    text_format.Parse(text_config, config_proto)
+
+    hello_world_packet = mp.packet_creator.create_string('hello world')
+    out = []
+    graph = mp.CalculatorGraph(graph_config=config_proto)
+    graph.observe_output_stream('out', lambda _, packet: out.append(packet))
+    graph.start_run()
+    graph.add_packet_to_input_stream(
+        stream='in', packet=hello_world_packet.at(0))
+    graph.wait_until_idle()
+    graph.add_packet_to_input_stream(
+        stream='in', packet=hello_world_packet.at(0))
+    with self.assertRaisesRegex(
+        ValueError, 'Current minimum expected timestamp is 1 but received 0.'):
+      graph.wait_until_idle()
+
+  def testSidePacketGraph(self):
+    text_config = """
+      node {
+        calculator: 'StringToUint64Calculator'
+        input_side_packet: "string"
+        output_side_packet: "number"
+      }
+    """
+    config_proto = calculator_pb2.CalculatorGraphConfig()
+    text_format.Parse(text_config, config_proto)
+    graph = mp.CalculatorGraph(graph_config=config_proto)
+    graph.start_run(
+        input_side_packets={'string': mp.packet_creator.create_string('42')})
+    graph.wait_until_done()
+    self.assertFalse(graph.has_error())
+    self.assertEqual(
+        mp.packet_getter.get_uint(graph.get_output_side_packet('number')), 42)
+
+
+if __name__ == '__main__':
+  absltest.main()

mediapipe/python/framework_bindings.cc

@@ -0,0 +1,39 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/python/pybind/calculator_graph.h"
+#include "mediapipe/python/pybind/image_frame.h"
+#include "mediapipe/python/pybind/matrix.h"
+#include "mediapipe/python/pybind/packet.h"
+#include "mediapipe/python/pybind/packet_creator.h"
+#include "mediapipe/python/pybind/packet_getter.h"
+#include "mediapipe/python/pybind/resource_util.h"
+#include "mediapipe/python/pybind/timestamp.h"
+
+namespace mediapipe {
+namespace python {
+
+PYBIND11_MODULE(_framework_bindings, m) {
+  ResourceUtilSubmodule(&m);
+  ImageFrameSubmodule(&m);
+  MatrixSubmodule(&m);
+  TimestampSubmodule(&m);
+  PacketSubmodule(&m);
+  PacketCreatorSubmodule(&m);
+  PacketGetterSubmodule(&m);
+  CalculatorGraphSubmodule(&m);
+}
+
+}  // namespace python
+}  // namespace mediapipe

mediapipe/python/image_frame_test.py

@@ -0,0 +1,145 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for mediapipe.python._framework_bindings.image_frame."""
+
+import random
+from absl.testing import absltest
+import cv2
+import numpy as np
+import PIL.Image
+import mediapipe.python as mp
+
+
+# TODO: Add unit tests specifically for memory management.
+class ImageFrameTest(absltest.TestCase):
+
+  def testCreateImageFrameFromGrayCvMat(self):
+    w, h = random.randrange(3, 100), random.randrange(3, 100)
+    mat = cv2.cvtColor(
+        np.random.randint(2**8 - 1, size=(h, w, 3), dtype=np.uint8),
+        cv2.COLOR_RGB2GRAY)
+    mat[2, 2] = 42
+    image_frame = mp.ImageFrame(image_format=mp.ImageFormat.GRAY8, data=mat)
+    self.assertTrue(np.array_equal(mat, image_frame.numpy_view()))
+    with self.assertRaisesRegex(IndexError, 'index dimension mismatch'):
+      print(image_frame[w, h, 1])
+    with self.assertRaisesRegex(IndexError, 'out of bounds'):
+      print(image_frame[w, h])
+    self.assertEqual(42, image_frame[2, 2])
+
+  def testCreateImageFrameFromRgbCvMat(self):
+    w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
+    mat = cv2.cvtColor(
+        np.random.randint(2**8 - 1, size=(h, w, channels), dtype=np.uint8),
+        cv2.COLOR_RGB2BGR)
+    mat[2, 2, 1] = 42
+    image_frame = mp.ImageFrame(image_format=mp.ImageFormat.SRGB, data=mat)
+    self.assertTrue(np.array_equal(mat, image_frame.numpy_view()))
+    with self.assertRaisesRegex(IndexError, 'out of bounds'):
+      print(image_frame[w, h, channels])
+    self.assertEqual(42, image_frame[2, 2, 1])
+
+  def testCreateImageFrameFromRgb48CvMat(self):
+    w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
+    mat = cv2.cvtColor(
+        np.random.randint(2**16 - 1, size=(h, w, channels), dtype=np.uint16),
+        cv2.COLOR_RGB2BGR)
+    mat[2, 2, 1] = 42
+    image_frame = mp.ImageFrame(image_format=mp.ImageFormat.SRGB48, data=mat)
+    self.assertTrue(np.array_equal(mat, image_frame.numpy_view()))
+    with self.assertRaisesRegex(IndexError, 'out of bounds'):
+      print(image_frame[w, h, channels])
+    self.assertEqual(42, image_frame[2, 2, 1])
+
+  def testCreateImageFrameFromGrayPilImage(self):
+    w, h = random.randrange(3, 100), random.randrange(3, 100)
+    img = PIL.Image.fromarray(
+        np.random.randint(2**8 - 1, size=(h, w), dtype=np.uint8), 'L')
+    image_frame = mp.ImageFrame(
+        image_format=mp.ImageFormat.GRAY8, data=np.asarray(img))
+    self.assertTrue(np.array_equal(np.asarray(img), image_frame.numpy_view()))
+    with self.assertRaisesRegex(IndexError, 'index dimension mismatch'):
+      print(image_frame[w, h, 1])
+    with self.assertRaisesRegex(IndexError, 'out of bounds'):
+      print(image_frame[w, h])
+
+  def testCreateImageFrameFromRgbPilImage(self):
+    w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
+    img = PIL.Image.fromarray(
+        np.random.randint(2**8 - 1, size=(h, w, channels), dtype=np.uint8),
+        'RGB')
+    image_frame = mp.ImageFrame(
+        image_format=mp.ImageFormat.SRGB, data=np.asarray(img))
+    self.assertTrue(np.array_equal(np.asarray(img), image_frame.numpy_view()))
+    with self.assertRaisesRegex(IndexError, 'out of bounds'):
+      print(image_frame[w, h, channels])
+
+  def testCreateImageFrameFromRgba64PilImage(self):
+    w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 4
+    img = PIL.Image.fromarray(
+        np.random.randint(2**16 - 1, size=(h, w, channels), dtype=np.uint16),
+        'RGBA')
+    image_frame = mp.ImageFrame(
+        image_format=mp.ImageFormat.SRGBA64,
+        data=np.asarray(img, dtype=np.uint16))
+    self.assertTrue(np.array_equal(np.asarray(img), image_frame.numpy_view()))
+    with self.assertRaisesRegex(IndexError, 'out of bounds'):
+      print(image_frame[1000, 1000, 1000])
+
+  def testImageFrameNumbyView(self):
+    w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
+    mat = cv2.cvtColor(
+        np.random.randint(2**8 - 1, size=(h, w, channels), dtype=np.uint8),
+        cv2.COLOR_RGB2BGR)
+    image_frame = mp.ImageFrame(image_format=mp.ImageFormat.SRGB, data=mat)
+    output_ndarray = image_frame.numpy_view()
+    self.assertTrue(np.array_equal(mat, image_frame.numpy_view()))
+    # The output of numpy_view() is a reference to the internal data and it's
+    # unwritable after creation.
+    with self.assertRaisesRegex(ValueError,
+                                'assignment destination is read-only'):
+      output_ndarray[0, 0, 0] = 0
+    copied_ndarray = np.copy(output_ndarray)
+    copied_ndarray[0, 0, 0] = 0
+
+  def testCroppedGray8Image(self):
+    w, h = random.randrange(20, 100), random.randrange(20, 100)
+    channels, offset = 3, 10
+    mat = cv2.cvtColor(
+        np.random.randint(2**8 - 1, size=(h, w, channels), dtype=np.uint8),
+        cv2.COLOR_RGB2GRAY)
+    image_frame = mp.ImageFrame(
+        image_format=mp.ImageFormat.GRAY8,
+        data=mat[offset:-offset, offset:-offset])
+    self.assertTrue(
+        np.array_equal(mat[offset:-offset, offset:-offset],
+                       image_frame.numpy_view()))
+
+  def testCroppedRGBImage(self):
+    w, h = random.randrange(20, 100), random.randrange(20, 100)
+    channels, offset = 3, 10
+    mat = cv2.cvtColor(
+        np.random.randint(2**8 - 1, size=(h, w, channels), dtype=np.uint8),
+        cv2.COLOR_RGB2BGR)
+    image_frame = mp.ImageFrame(
+        image_format=mp.ImageFormat.SRGB,
+        data=mat[offset:-offset, offset:-offset, :])
+    self.assertTrue(
+        np.array_equal(mat[offset:-offset, offset:-offset, :],
+                       image_frame.numpy_view()))
+
+
+if __name__ == '__main__':
+  absltest.main()

mediapipe/python/packet_creator.py

@@ -0,0 +1,126 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""The public facing packet creator APIs."""
+
+from typing import List, Union
+
+import numpy as np
+
+from google.protobuf import message
+from mediapipe.python._framework_bindings import _packet_creator
+from mediapipe.python._framework_bindings import image_frame
+from mediapipe.python._framework_bindings import packet
+
+
+create_string = _packet_creator.create_string
+create_bool = _packet_creator.create_bool
+create_int = _packet_creator.create_int
+create_int8 = _packet_creator.create_int8
+create_int16 = _packet_creator.create_int16
+create_int32 = _packet_creator.create_int32
+create_int64 = _packet_creator.create_int64
+create_uint8 = _packet_creator.create_uint8
+create_uint16 = _packet_creator.create_uint16
+create_uint32 = _packet_creator.create_uint32
+create_uint64 = _packet_creator.create_uint64
+create_float = _packet_creator.create_float
+create_double = _packet_creator.create_double
+create_int_array = _packet_creator.create_int_array
+create_float_array = _packet_creator.create_float_array
+create_int_vector = _packet_creator.create_int_vector
+create_float_vector = _packet_creator.create_float_vector
+create_string_vector = _packet_creator.create_string_vector
+create_packet_vector = _packet_creator.create_packet_vector
+create_string_to_packet_map = _packet_creator.create_string_to_packet_map
+create_matrix = _packet_creator.create_matrix
+
+
+def create_image_frame(
+    data: Union[image_frame.ImageFrame, np.ndarray],
+    *,
+    image_format: image_frame.ImageFormat = None) -> packet.Packet:
+  """Create a MediaPipe ImageFrame packet.
+
+  A MediaPipe ImageFrame packet can be created from either the raw pixel data
+  represented as a numpy array with one of the uint8, uint16, and float data
+  types or an existing MediaPipe ImageFrame object. The data will be realigned
+  and copied into an ImageFrame object inside of the packet.
+
+  Args:
+    data: A MediaPipe ImageFrame object or the raw pixel data that is
+      represnted as a numpy ndarray.
+    image_format: One of the image_frame.ImageFormat enum types.
+
+  Returns:
+    A MediaPipe ImageFrame Packet.
+
+  Raises:
+    ValueError:
+      i) When "data" is a numpy ndarray, "image_format" is not provided.
+      ii) When "data" is an ImageFrame object, the "image_format" arg doesn't
+        match the image format of the "data" ImageFrame object.
+    TypeError: If "image format" doesn't match "data" array's data type.
+
+  Examples:
+    np_array = np.random.randint(255, size=(321, 123, 3), dtype=np.uint8)
+    image_frame_packet = mp.packet_creator.create_image_frame(
+        image_format=mp.ImageFormat.SRGB, data=np_array)
+
+    image_frame = mp.ImageFrame(image_format=mp.ImageFormat.SRGB, data=np_array)
+    image_frame_packet = mp.packet_creator.create_image_frame(image_frame)
+
+  """
+  if isinstance(data, image_frame.ImageFrame):
+    if image_format is not None and data.image_format != image_format:
+      raise ValueError(
+          'The provided image_format doesn\'t match the one from the data arg.')
+    # pylint:disable=protected-access
+    return _packet_creator._create_image_frame_with_copy(data)
+    # pylint:enable=protected-access
+  else:
+    if image_format is None:
+      raise ValueError('Please provide \'image_format\' with \'data\'.')
+    # pylint:disable=protected-access
+    return _packet_creator._create_image_frame_with_copy(image_format, data)
+    # pylint:enable=protected-access
+
+
+def create_proto(proto_message: message.Message) -> packet.Packet:
+  """Create a MediaPipe protobuf message packet.
+
+  Args:
+    proto_message: A Python protobuf message.
+
+  Returns:
+    A MediaPipe protobuf message Packet.
+
+  Raises:
+    RuntimeError: If the protobuf message type is not registered in MediaPipe.
+
+  Examples:
+    detection = detection_pb2.Detection()
+    text_format.Parse('score: 0.5', detection)
+    packet = mp.packet_creator.create_proto(detection)
+    output_detection = mp.packet_getter.get_proto(packet)
+  """
+  # pylint:disable=protected-access
+  return _packet_creator._create_proto(proto_message.DESCRIPTOR.full_name,
+                                       proto_message.SerializeToString())
+  # pylint:enable=protected-access
+
+
+def create_proto_vector(message_list: List[message.Message]) -> packet.Packet:
+  raise NotImplementedError('create_proto_vector is not implemented.')

mediapipe/python/packet_getter.py

@@ -0,0 +1,117 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""The public facing packet getter APIs."""
+
+from typing import List, Type
+
+from google.protobuf import message
+from google.protobuf import symbol_database
+from mediapipe.python._framework_bindings import _packet_getter
+from mediapipe.python._framework_bindings import packet as mp_packet
+
+get_str = _packet_getter.get_str
+get_bytes = _packet_getter.get_bytes
+get_bool = _packet_getter.get_bool
+get_int = _packet_getter.get_int
+get_uint = _packet_getter.get_uint
+get_float = _packet_getter.get_float
+get_int_list = _packet_getter.get_int_list
+get_float_list = _packet_getter.get_float_list
+get_str_list = _packet_getter.get_str_list
+get_packet_list = _packet_getter.get_packet_list
+get_str_to_packet_dict = _packet_getter.get_str_to_packet_dict
+get_image_frame = _packet_getter.get_image_frame
+get_matrix = _packet_getter.get_matrix
+
+
+def get_proto(packet: mp_packet.Packet) -> Type[message.Message]:
+  """Get the content of a MediaPipe proto Packet as a proto message.
+
+  Args:
+    packet: A MediaPipe proto Packet.
+
+  Returns:
+    A proto message.
+
+  Raises:
+    TypeError: If the message descriptor can't be found by type name.
+
+  Examples:
+    detection = detection_pb2.Detection()
+    text_format.Parse('score: 0.5', detection)
+    proto_packet = mp.packet_creator.create_proto(detection)
+    output_proto = mp.packet_getter.get_proto(proto_packet)
+  """
+  # pylint:disable=protected-access
+  proto_type_name = _packet_getter._get_proto_type_name(packet)
+  # pylint:enable=protected-access
+  try:
+    descriptor = symbol_database.Default().pool.FindMessageTypeByName(
+        proto_type_name)
+  except KeyError:
+    raise TypeError('Can not find message descriptor by type name: %s' %
+                    proto_type_name)
+
+  message_class = symbol_database.Default().GetPrototype(descriptor)
+  # pylint:disable=protected-access
+  serialized_proto = _packet_getter._get_serialized_proto(packet)
+  # pylint:enable=protected-access
+  proto_message = message_class()
+  proto_message.ParseFromString(serialized_proto)
+  return proto_message
+
+
+def get_proto_list(packet: mp_packet.Packet) -> List[message.Message]:
+  """Get the content of a MediaPipe proto vector Packet as a proto message list.
+
+  Args:
+    packet: A MediaPipe proto vector Packet.
+
+  Returns:
+    A proto message list.
+
+  Raises:
+    TypeError: If the message descriptor can't be found by type name.
+
+  Examples:
+    proto_list = mp.packet_getter.get_proto_list(protos_packet)
+  """
+  # pylint:disable=protected-access
+  vector_size = _packet_getter._get_proto_vector_size(packet)
+  # pylint:enable=protected-access
+  # Return empty list if the proto vector is empty.
+  if vector_size == 0:
+    return []
+
+  # pylint:disable=protected-access
+  proto_type_name = _packet_getter._get_proto_vector_element_type_name(packet)
+  # pylint:enable=protected-access
+  try:
+    descriptor = symbol_database.Default().pool.FindMessageTypeByName(
+        proto_type_name)
+  except KeyError:
+    raise TypeError('Can not find message descriptor by type name: %s' %
+                    proto_type_name)
+  message_class = symbol_database.Default().GetPrototype(descriptor)
+  # pylint:disable=protected-access
+  serialized_protos = _packet_getter._get_serialized_proto_list(packet)
+  # pylint:enable=protected-access
+  proto_message_list = []
+  for serialized_proto in serialized_protos:
+    proto_message = message_class()
+    proto_message.ParseFromString(serialized_proto)
+    proto_message_list.append(proto_message)
+  return proto_message_list

mediapipe/python/packet_test.py

@@ -0,0 +1,349 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for mediapipe.python._framework_bindings.packet."""
+
+import gc
+import random
+import sys
+from absl.testing import absltest
+import numpy as np
+import mediapipe.python as mp
+from google.protobuf import text_format
+from mediapipe.framework.formats import detection_pb2
+
+
+class PacketTest(absltest.TestCase):
+
+  def testEmptyPacket(self):
+    p = mp.Packet()
+    self.assertTrue(p.is_empty())
+
+  def testBooleanPacket(self):
+    p = mp.packet_creator.create_bool(True)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_bool(p), True)
+    self.assertEqual(p.timestamp, 0)
+
+  def testIntPacket(self):
+    with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
+      p = mp.packet_creator.create_int(2**32)
+    p = mp.packet_creator.create_int(42)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p), 42)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_int(np.intc(1))
+    p2.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p2), 1)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testInt8Packet(self):
+    with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
+      p = mp.packet_creator.create_int8(2**7)
+    p = mp.packet_creator.create_int8(2**7 - 1)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p), 2**7 - 1)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_int8(np.int8(1))
+    p2.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p2), 1)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testInt16Packet(self):
+    with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
+      p = mp.packet_creator.create_int16(2**15)
+    p = mp.packet_creator.create_int16(2**15 - 1)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p), 2**15 - 1)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_int16(np.int16(1))
+    p2.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p2), 1)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testInt32Packet(self):
+    with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
+      p = mp.packet_creator.create_int32(2**31)
+
+    p = mp.packet_creator.create_int32(2**31 - 1)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p), 2**31 - 1)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_int32(np.int32(1))
+    p2.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p2), 1)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testInt64Packet(self):
+    p = mp.packet_creator.create_int64(2**63 - 1)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p), 2**63 - 1)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_int64(np.int64(1))
+    p2.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_int(p2), 1)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testUint8Packet(self):
+    with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
+      p = mp.packet_creator.create_uint8(2**8)
+    p = mp.packet_creator.create_uint8(2**8 - 1)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_uint(p), 2**8 - 1)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_uint8(np.uint8(1))
+    p2.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_uint(p2), 1)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testUint16Packet(self):
+    with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
+      p = mp.packet_creator.create_uint16(2**16)
+    p = mp.packet_creator.create_uint16(2**16 - 1)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_uint(p), 2**16 - 1)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_uint16(np.uint16(1))
+    p2.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_uint(p2), 1)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testUint32Packet(self):
+    with self.assertRaisesRegex(OverflowError, 'execeeds the maximum value'):
+      p = mp.packet_creator.create_uint32(2**32)
+    p = mp.packet_creator.create_uint32(2**32 - 1)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_uint(p), 2**32 - 1)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_uint32(np.uint32(1))
+    p2.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_uint(p2), 1)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testUint64Packet(self):
+    p = mp.packet_creator.create_uint64(2**64 - 1)
+    p.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_uint(p), 2**64 - 1)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_uint64(np.uint64(1))
+    p2.timestamp = 0
+    self.assertEqual(mp.packet_getter.get_uint(p2), 1)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testFloatPacket(self):
+    p = mp.packet_creator.create_float(0.42)
+    p.timestamp = 0
+    self.assertAlmostEqual(mp.packet_getter.get_float(p), 0.42)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_float(np.float(0.42))
+    p2.timestamp = 0
+    self.assertAlmostEqual(mp.packet_getter.get_float(p2), 0.42)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testDoublePacket(self):
+    p = mp.packet_creator.create_double(0.42)
+    p.timestamp = 0
+    self.assertAlmostEqual(mp.packet_getter.get_float(p), 0.42)
+    self.assertEqual(p.timestamp, 0)
+    p2 = mp.packet_creator.create_double(np.double(0.42))
+    p2.timestamp = 0
+    self.assertAlmostEqual(mp.packet_getter.get_float(p2), 0.42)
+    self.assertEqual(p2.timestamp, 0)
+
+  def testDetectionProtoPacket(self):
+    detection = detection_pb2.Detection()
+    text_format.Parse('score: 0.5', detection)
+    p = mp.packet_creator.create_proto(detection).at(100)
+
+  def testStringPacket(self):
+    p = mp.packet_creator.create_string('abc').at(100)
+    self.assertEqual(mp.packet_getter.get_str(p), 'abc')
+    self.assertEqual(p.timestamp, 100)
+    p.timestamp = 200
+    self.assertEqual(p.timestamp, 200)
+
+  def testBytesPacket(self):
+    p = mp.packet_creator.create_string(b'xd0\xba\xd0').at(300)
+    self.assertEqual(mp.packet_getter.get_bytes(p), b'xd0\xba\xd0')
+    self.assertEqual(p.timestamp, 300)
+
+  def testIntArrayPacket(self):
+    p = mp.packet_creator.create_int_array([1, 2, 3]).at(100)
+    self.assertEqual(p.timestamp, 100)
+
+  def testFloatArrayPacket(self):
+    p = mp.packet_creator.create_float_array([0.1, 0.2, 0.3]).at(100)
+    self.assertEqual(p.timestamp, 100)
+
+  def testIntVectorPacket(self):
+    p = mp.packet_creator.create_int_vector([1, 2, 3]).at(100)
+    self.assertEqual(mp.packet_getter.get_int_list(p), [1, 2, 3])
+    self.assertEqual(p.timestamp, 100)
+
+  def testFloatVectorPacket(self):
+    p = mp.packet_creator.create_float_vector([0.1, 0.2, 0.3]).at(100)
+    output_list = mp.packet_getter.get_float_list(p)
+    self.assertAlmostEqual(output_list[0], 0.1)
+    self.assertAlmostEqual(output_list[1], 0.2)
+    self.assertAlmostEqual(output_list[2], 0.3)
+    self.assertEqual(p.timestamp, 100)
+
+  def testStringVectorPacket(self):
+    p = mp.packet_creator.create_string_vector(['a', 'b', 'c']).at(100)
+    output_list = mp.packet_getter.get_str_list(p)
+    self.assertEqual(output_list[0], 'a')
+    self.assertEqual(output_list[1], 'b')
+    self.assertEqual(output_list[2], 'c')
+    self.assertEqual(p.timestamp, 100)
+
+  def testPacketVectorPacket(self):
+    p = mp.packet_creator.create_packet_vector([
+        mp.packet_creator.create_float(0.42),
+        mp.packet_creator.create_int(42),
+        mp.packet_creator.create_string('42')
+    ]).at(100)
+    output_list = mp.packet_getter.get_packet_list(p)
+    self.assertAlmostEqual(mp.packet_getter.get_float(output_list[0]), 0.42)
+    self.assertEqual(mp.packet_getter.get_int(output_list[1]), 42)
+    self.assertEqual(mp.packet_getter.get_str(output_list[2]), '42')
+    self.assertEqual(p.timestamp, 100)
+
+  def testStringToPacketMapPacket(self):
+    p = mp.packet_creator.create_string_to_packet_map({
+        'float': mp.packet_creator.create_float(0.42),
+        'int': mp.packet_creator.create_int(42),
+        'string': mp.packet_creator.create_string('42')
+    }).at(100)
+    output_list = mp.packet_getter.get_str_to_packet_dict(p)
+    self.assertAlmostEqual(
+        mp.packet_getter.get_float(output_list['float']), 0.42)
+    self.assertEqual(mp.packet_getter.get_int(output_list['int']), 42)
+    self.assertEqual(mp.packet_getter.get_str(output_list['string']), '42')
+    self.assertEqual(p.timestamp, 100)
+
+  def testUint8ImageFramePacket(self):
+    uint8_img = np.random.randint(
+        2**8 - 1,
+        size=(random.randrange(3, 100), random.randrange(3, 100), 3),
+        dtype=np.uint8)
+    p = mp.packet_creator.create_image_frame(
+        mp.ImageFrame(image_format=mp.ImageFormat.SRGB, data=uint8_img))
+    output_image_frame = mp.packet_getter.get_image_frame(p)
+    self.assertTrue(np.array_equal(output_image_frame.numpy_view(), uint8_img))
+
+  def testUint16ImageFramePacket(self):
+    uint16_img = np.random.randint(
+        2**16 - 1,
+        size=(random.randrange(3, 100), random.randrange(3, 100), 4),
+        dtype=np.uint16)
+    p = mp.packet_creator.create_image_frame(
+        mp.ImageFrame(image_format=mp.ImageFormat.SRGBA64, data=uint16_img))
+    output_image_frame = mp.packet_getter.get_image_frame(p)
+    self.assertTrue(np.array_equal(output_image_frame.numpy_view(), uint16_img))
+
+  def testFloatImageFramePacket(self):
+    float_img = np.float32(
+        np.random.random_sample(
+            (random.randrange(3, 100), random.randrange(3, 100), 2)))
+    p = mp.packet_creator.create_image_frame(
+        mp.ImageFrame(image_format=mp.ImageFormat.VEC32F2, data=float_img))
+    output_image_frame = mp.packet_getter.get_image_frame(p)
+    self.assertTrue(np.allclose(output_image_frame.numpy_view(), float_img))
+
+  def testImageFramePacketCreationCopyMode(self):
+    w, h, channels = random.randrange(3, 100), random.randrange(3, 100), 3
+    rgb_data = np.random.randint(255, size=(h, w, channels), dtype=np.uint8)
+    # rgb_data is c_contiguous.
+    self.assertTrue(rgb_data.flags.c_contiguous)
+    initial_ref_count = sys.getrefcount(rgb_data)
+    p = mp.packet_creator.create_image_frame(
+        image_format=mp.ImageFormat.SRGB, data=rgb_data)
+    # copy mode doesn't increase the ref count of the data.
+    self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
+
+    rgb_data = rgb_data[:, :, ::-1]
+    # rgb_data is now not c_contiguous. But, copy mode shouldn't be affected.
+    self.assertFalse(rgb_data.flags.c_contiguous)
+    initial_ref_count = sys.getrefcount(rgb_data)
+    p = mp.packet_creator.create_image_frame(
+        image_format=mp.ImageFormat.SRGB, data=rgb_data)
+    # copy mode doesn't increase the ref count of the data.
+    self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
+
+    output_frame = mp.packet_getter.get_image_frame(p)
+    self.assertEqual(output_frame.height, h)
+    self.assertEqual(output_frame.width, w)
+    self.assertEqual(output_frame.channels, channels)
+    self.assertTrue(np.array_equal(output_frame.numpy_view(), rgb_data))
+
+    del p
+    del output_frame
+    gc.collect()
+    # Destroying the packet also doesn't affect the ref count becuase of the
+    # copy mode.
+    self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
+
+  def testImageFramePacketCopyConstuctionWithCropping(self):
+    w, h, channels = random.randrange(40, 100), random.randrange(40, 100), 3
+    channels, offset = 3, 10
+    rgb_data = np.random.randint(255, size=(h, w, channels), dtype=np.uint8)
+    initial_ref_count = sys.getrefcount(rgb_data)
+    p = mp.packet_creator.create_image_frame(
+        image_format=mp.ImageFormat.SRGB,
+        data=rgb_data[offset:-offset, offset:-offset, :])
+    # copy mode doesn't increase the ref count of the data.
+    self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
+    output_frame = mp.packet_getter.get_image_frame(p)
+    self.assertEqual(output_frame.height, h - 2 * offset)
+    self.assertEqual(output_frame.width, w - 2 * offset)
+    self.assertEqual(output_frame.channels, channels)
+    self.assertTrue(
+        np.array_equal(rgb_data[offset:-offset, offset:-offset, :],
+                       output_frame.numpy_view()))
+    del p
+    del output_frame
+    gc.collect()
+    # Destroying the packet also doesn't affect the ref count becuase of the
+    # copy mode.
+    self.assertEqual(sys.getrefcount(rgb_data), initial_ref_count)
+
+  def testMatrixPacket(self):
+    np_matrix = np.array([[.1, .2, .3], [.4, .5, .6]])
+    initial_ref_count = sys.getrefcount(np_matrix)
+    p = mp.packet_creator.create_matrix(np_matrix)
+    # Copy mode should not increase the ref count of np_matrix.
+    self.assertEqual(initial_ref_count, sys.getrefcount(np_matrix))
+    output_matrix = mp.packet_getter.get_matrix(p)
+    del np_matrix
+    gc.collect()
+    self.assertTrue(
+        np.allclose(output_matrix, np.array([[.1, .2, .3], [.4, .5, .6]])))
+
+  def testMatrixPacketWithNonCContiguousData(self):
+    np_matrix = np.array([[.1, .2, .3], [.4, .5, .6]])[:, ::-1]
+    # np_matrix is not c_contiguous.
+    self.assertFalse(np_matrix.flags.c_contiguous)
+    p = mp.packet_creator.create_matrix(np_matrix)
+    initial_ref_count = sys.getrefcount(np_matrix)
+    # Copy mode should not increase the ref count of np_matrix.
+    self.assertEqual(initial_ref_count, sys.getrefcount(np_matrix))
+    output_matrix = mp.packet_getter.get_matrix(p)
+    del np_matrix
+    gc.collect()
+    self.assertTrue(
+        np.allclose(output_matrix,
+                    np.array([[.1, .2, .3], [.4, .5, .6]])[:, ::-1]))
+
+if __name__ == '__main__':
+  absltest.main()

mediapipe/python/pybind/BUILD

@@ -0,0 +1,141 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@pybind11_bazel//:build_defs.bzl", "pybind_library")
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/python:__subpackages__"])
+
+pybind_library(
+    name = "calculator_graph",
+    srcs = ["calculator_graph.cc"],
+    hdrs = ["calculator_graph.h"],
+    deps = [
+        ":util",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_graph",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:map_util",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:calculator_graph_template_cc_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+pybind_library(
+    name = "image_frame",
+    srcs = ["image_frame.cc"],
+    hdrs = ["image_frame.h"],
+    deps = [
+        ":image_frame_util",
+        ":util",
+    ],
+)
+
+pybind_library(
+    name = "image_frame_util",
+    hdrs = ["image_frame_util.h"],
+    deps = [
+        "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:logging",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+pybind_library(
+    name = "matrix",
+    srcs = ["matrix.cc"],
+    hdrs = ["matrix.h"],
+    deps = [
+        "//mediapipe/framework/formats:matrix",
+    ],
+)
+
+pybind_library(
+    name = "packet",
+    srcs = ["packet.cc"],
+    hdrs = ["packet.h"],
+    deps = [
+        ":util",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework:timestamp",
+    ],
+)
+
+pybind_library(
+    name = "packet_creator",
+    srcs = ["packet_creator.cc"],
+    hdrs = ["packet_creator.h"],
+    deps = [
+        ":image_frame_util",
+        ":util",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:integral_types",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+pybind_library(
+    name = "packet_getter",
+    srcs = ["packet_getter.cc"],
+    hdrs = ["packet_getter.h"],
+    deps = [
+        ":image_frame_util",
+        ":util",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:integral_types",
+    ],
+)
+
+pybind_library(
+    name = "timestamp",
+    srcs = ["timestamp.cc"],
+    hdrs = ["timestamp.h"],
+    deps = [
+        ":util",
+        "//mediapipe/framework:timestamp",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+pybind_library(
+    name = "resource_util",
+    srcs = ["resource_util.cc"],
+    hdrs = ["resource_util.h"],
+    deps = [
+        "//mediapipe/util:resource_util",
+        "@com_google_absl//absl/flags:flag",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+pybind_library(
+    name = "util",
+    hdrs = ["util.h"],
+    deps = [
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:status",
+    ],
+)

mediapipe/python/pybind/calculator_graph.cc

@@ -0,0 +1,466 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/python/pybind/calculator_graph.h"
+
+#include "absl/memory/memory.h"
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_graph.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/map_util.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/tool/calculator_graph_template.pb.h"
+#include "mediapipe/python/pybind/util.h"
+#include "pybind11/embed.h"
+#include "pybind11/pybind11.h"
+#include "pybind11/stl.h"
+
+namespace mediapipe {
+namespace python {
+
+template <typename T>
+T ParseProto(const py::object& proto_object) {
+  T proto;
+  if (!ParseTextProto<T>(proto_object.str(), &proto)) {
+    throw RaisePyError(
+        PyExc_RuntimeError,
+        absl::StrCat("Failed to parse: ", std::string(proto_object.str()))
+            .c_str());
+  }
+  return proto;
+}
+
+namespace py = pybind11;
+
+void CalculatorGraphSubmodule(pybind11::module* module) {
+  py::module m = module->def_submodule("calculator_graph",
+                                       "MediaPipe calculator graph module.");
+
+  using GraphInputStreamAddMode =
+      mediapipe::CalculatorGraph::GraphInputStreamAddMode;
+
+  py::enum_<GraphInputStreamAddMode>(m, "GraphInputStreamAddMode")
+      .value("WAIT_TILL_NOT_FULL", GraphInputStreamAddMode::WAIT_TILL_NOT_FULL)
+      .value("ADD_IF_NOT_FULL", GraphInputStreamAddMode::ADD_IF_NOT_FULL)
+      .export_values();
+
+  // Calculator Graph
+  py::class_<CalculatorGraph> calculator_graph(
+      m, "CalculatorGraph", R"doc(The primary API for the MediaPipe Framework.
+
+  MediaPipe processing takes place inside a graph, which defines packet flow
+  paths between nodes. A graph can have any number of inputs and outputs, and
+  data flow can branch and merge. Generally data flows forward, but backward
+  loops are possible.)doc");
+
+  // TODO: Support graph initialization with graph templates and
+  // subgraph.
+  calculator_graph.def(
+      py::init([](py::args args, py::kwargs kwargs) {
+        if (!args.empty()) {
+          throw RaisePyError(PyExc_RuntimeError,
+                             "Invalid position input arguments.");
+        }
+        bool init_with_binary_graph = false;
+        bool init_with_graph_proto = false;
+        CalculatorGraphConfig graph_config_proto;
+        for (const auto& kw : kwargs) {
+          const std::string& key = kw.first.cast<std::string>();
+          if (key == "binary_graph_path") {
+            init_with_binary_graph = true;
+            std::string file_name(kw.second.cast<py::object>().str());
+            auto status = file::Exists(file_name);
+            if (!status.ok()) {
+              throw RaisePyError(PyExc_FileNotFoundError,
+                                 status.message().data());
+            }
+            std::string graph_config_string;
+            RaisePyErrorIfNotOk(
+                file::GetContents(file_name, &graph_config_string));
+            if (!graph_config_proto.ParseFromArray(
+                    graph_config_string.c_str(),
+                    graph_config_string.length())) {
+              throw RaisePyError(
+                  PyExc_RuntimeError,
+                  absl::StrCat("Failed to parse the binary graph: ", file_name)
+                      .c_str());
+            }
+          } else if (key == "graph_config") {
+            init_with_graph_proto = true;
+            graph_config_proto =
+                ParseProto<CalculatorGraphConfig>(kw.second.cast<py::object>());
+          } else {
+            throw RaisePyError(
+                PyExc_RuntimeError,
+                absl::StrCat("Unknown kwargs input argument: ", key).c_str());
+          }
+        }
+
+        if (!(init_with_binary_graph ^ init_with_graph_proto)) {
+          throw RaisePyError(
+              PyExc_ValueError,
+              "Please provide \'binary_graph\' to initialize the graph with"
+              " binary graph or provide \'graph_config\' to initialize the "
+              " with graph config proto.");
+        }
+        auto calculator_graph = absl::make_unique<CalculatorGraph>();
+        RaisePyErrorIfNotOk(calculator_graph->Initialize(graph_config_proto));
+        return calculator_graph.release();
+      }),
+      R"doc(Initialize CalculatorGraph object.
+
+  Args:
+    binary_graph_path: The path to a binary mediapipe graph file (.binarypb).
+    graph_config: A single CalculatorGraphConfig proto message or its text proto
+      format.
+
+  Raises:
+    FileNotFoundError: If the binary graph file can't be found.
+    ValueError: If the input arguments prvoided are more than needed or the
+      graph validation process contains error.
+)doc");
+
+  // TODO: Return a Python CalculatorGraphConfig instead.
+  calculator_graph.def_property_readonly(
+      "config",
+      [](const CalculatorGraph& self) { return self.Config().DebugString(); });
+
+  calculator_graph.def_property_readonly(
+      "serialized_config", [](const CalculatorGraph& self) {
+        return py::bytes(self.Config().SerializeAsString());
+      });
+
+  calculator_graph.def_property_readonly(
+      "max_queue_size",
+      [](CalculatorGraph* self) { return self->GetMaxInputStreamQueueSize(); });
+
+  calculator_graph.def_property(
+      "graph_input_stream_add_mode",
+      [](const CalculatorGraph& self) {
+        return self.GetGraphInputStreamAddMode();
+      },
+      [](CalculatorGraph* self, CalculatorGraph::GraphInputStreamAddMode mode) {
+        self->SetGraphInputStreamAddMode(mode);
+      });
+
+  calculator_graph.def(
+      "add_packet_to_input_stream",
+      [](CalculatorGraph* self, const std::string& stream, const Packet& packet,
+         const Timestamp& timestamp) {
+        Timestamp packet_timestamp =
+            timestamp == Timestamp::Unset() ? packet.Timestamp() : timestamp;
+        if (!packet_timestamp.IsAllowedInStream()) {
+          throw RaisePyError(
+              PyExc_ValueError,
+              absl::StrCat(packet_timestamp.DebugString(),
+                           " can't be the timestamp of a Packet in a stream.")
+                  .c_str());
+        }
+        RaisePyErrorIfNotOk(
+            self->AddPacketToInputStream(stream, packet.At(packet_timestamp)));
+      },
+      R"doc(Add a packet to a graph input stream.
+
+  If the graph input stream add mode is ADD_IF_NOT_FULL, the packet will not be
+  added if any queue exceeds the max queue size specified by the graph config
+  and will raise a Python runtime error. The WAIT_TILL_NOT_FULL mode (default)
+  will block until the queues fall below the max queue size before adding the
+  packet. If the mode is max queue size is -1, then the packet is added
+  regardless of the sizes of the queues in the graph. The input stream must have
+  been specified in the configuration as a graph level input stream. On error,
+  nothing is added.
+
+  Args:
+    stream: The name of the graph input stream.
+    packet: The packet to be added into the input stream.
+    timestamp: The timestamp of the packet. If set, the original packet
+      timestamp will be overwritten.
+
+  Raises:
+    RuntimeError: If the stream is not a graph input stream or the packet can't
+      be added into the input stream due to the limited queue size or the wrong
+      packet type.
+    ValueError: If the timestamp of the Packet is invalid to be the timestamp of
+      a Packet in a stream.
+
+  Examples:
+    graph.add_packet_to_input_stream(
+        stream='in',
+        packet=packet_creator.create_string('hello world').at(0))
+
+    graph.add_packet_to_input_stream(
+        stream='in',
+        packet=packet_creator.create_string('hello world'),
+        timstamp=1)
+)doc",
+      py::arg("stream"), py::arg("packet"),
+      py::arg("timestamp") = Timestamp::Unset());
+
+  calculator_graph.def(
+      "close_input_stream",
+      [](CalculatorGraph* self, const std::string& stream) {
+        RaisePyErrorIfNotOk(self->CloseInputStream(stream));
+      },
+      R"doc(Close the named graph input stream.
+
+  Args:
+    stream: The name of the stream to be closed.
+
+  Raises:
+    RuntimeError: If the stream is not a graph input stream.
+
+)doc");
+
+  calculator_graph.def(
+      "close_all_packet_sources",
+      [](CalculatorGraph* self) {
+        RaisePyErrorIfNotOk(self->CloseAllPacketSources());
+      },
+      R"doc(Closes all the graph input streams and source calculator nodes.)doc");
+
+  calculator_graph.def(
+      "start_run",
+      [](CalculatorGraph* self, const pybind11::dict& input_side_packets) {
+        std::map<std::string, Packet> input_side_packet_map;
+        for (const auto& kv_pair : input_side_packets) {
+          InsertIfNotPresent(&input_side_packet_map,
+                             kv_pair.first.cast<std::string>(),
+                             kv_pair.second.cast<Packet>());
+        }
+        RaisePyErrorIfNotOk(self->StartRun(input_side_packet_map));
+      },
+
+      R"doc(Start a run of the calculator graph.
+
+  A non-blocking call to start a run of the graph and will return when the graph
+  is started. If input_side_packets is provided, the method will runs the graph
+  after adding the given extra input side packets.
+
+  start_run(), wait_until_done(), has_error(), add_packet_to_input_stream(), and
+  close() allow more control over the execution of the graph run.  You can
+  insert packets directly into a stream while the graph is running.
+  Once start_run() has been called, the graph will continue to run until
+  wait_until_done() is called.
+
+  If start_run() returns an error, then the graph is not started and a
+  subsequent call to start_run() can be attempted.
+
+  Args:
+    input_side_packets: A dict maps from the input side packet names to the
+      packets.
+
+  Raises:
+    RuntimeError: If the start run occurs any error, e.g. the graph config has
+      errors, the calculator can't be found, and the streams are not properly
+      connected.
+
+  Examples:
+    graph = mp.CalculatorGraph(graph_config=video_process_graph)
+    graph.start_run(
+        input_side_packets={
+            'input_path': packet_creator.create_string('/tmp/input.video'),
+            'output_path': packet_creator.create_string('/tmp/output.video')
+        })
+    graph.close()
+
+    out = []
+    graph = mp.CalculatorGraph(graph_config=pass_through_graph)
+    graph.observe_output_stream('out',
+                                lambda stream_name, packet: out.append(packet))
+    graph.start_run()
+    graph.add_packet_to_input_stream(
+        stream='in', packet=packet_creator.create_int(0), timestamp=0)
+    graph.add_packet_to_input_stream(
+        stream='in', packet=packet_creator.create_int(1), timestamp=1)
+    graph.close()
+
+)doc",
+      py::arg("input_side_packets") = (py::dict){});
+
+  calculator_graph.def(
+      "wait_until_done",
+      [](CalculatorGraph* self) { RaisePyErrorIfNotOk(self->WaitUntilDone()); },
+      R"doc(Wait for the current run to finish.
+
+  A blocking call to wait for the current run to finish (block the current
+  thread until all source calculators are stopped, all graph input streams have
+  been closed, and no more calculators can be run). This function can be called
+  only after start_run(),
+
+  Raises:
+    RuntimeError: If the graph occurs any error during the wait call.
+
+  Examples:
+    out = []
+    graph = mp.CalculatorGraph(graph_config=pass_through_graph)
+    graph.observe_output_stream('out', lambda stream_name, packet: out.append(packet))
+    graph.start_run()
+    graph.add_packet_to_input_stream(
+        stream='in', packet=packet_creator.create_int(0), timestamp=0)
+    graph.close_all_packet_sources()
+    graph.wait_until_done()
+
+)doc");
+
+  calculator_graph.def(
+      "wait_until_idle",
+      [](CalculatorGraph* self) { RaisePyErrorIfNotOk(self->WaitUntilIdle()); },
+      R"doc(Wait until the running graph is in the idle mode.
+
+  Wait until the running graph is in the idle mode, which is when nothing can
+  be scheduled and nothing is running in the worker threads. This function can
+  be called only after start_run().
+
+  NOTE: The graph must not have any source nodes because source nodes prevent
+  the running graph from becoming idle until the source nodes are done.
+
+  Raises:
+    RuntimeError: If the graph occurs any error during the wait call.
+
+  Examples:
+    out = []
+    graph = mp.CalculatorGraph(graph_config=pass_through_graph)
+    graph.observe_output_stream('out',
+                                lambda stream_name, packet: out.append(packet))
+    graph.start_run()
+    graph.add_packet_to_input_stream(
+        stream='in', packet=packet_creator.create_int(0), timestamp=0)
+    graph.wait_until_idle()
+
+)doc");
+
+  calculator_graph.def(
+      "wait_for_observed_output",
+      [](CalculatorGraph* self) {
+        RaisePyErrorIfNotOk(self->WaitForObservedOutput());
+      },
+      R"doc(Wait until a packet is emitted on one of the observed output streams.
+
+  Returns immediately if a packet has already been emitted since the last
+  call to this function.
+
+  Raises:
+    RuntimeError:
+      If the graph occurs any error or the graph is terminated while waiting.
+
+  Examples:
+    out = []
+    graph = mp.CalculatorGraph(graph_config=pass_through_graph)
+    graph.observe_output_stream('out',
+                                lambda stream_name, packet: out.append(packet))
+    graph.start_run()
+    graph.add_packet_to_input_stream(
+        stream='in', packet=packet_creator.create_int(0), timestamp=0)
+    graph.wait_for_observed_output()
+    value = packet_getter.get_int(out[0])
+    graph.add_packet_to_input_stream(
+        stream='in', packet=packet_creator.create_int(1), timestamp=1)
+    graph.wait_for_observed_output()
+    value = packet_getter.get_int(out[1])
+
+)doc");
+
+  calculator_graph.def(
+      "has_error", [](const CalculatorGraph& self) { return self.HasError(); },
+      R"doc(Quick non-locking means of checking if the graph has encountered an error)doc");
+
+  calculator_graph.def(
+      "get_combined_error_message",
+      [](CalculatorGraph* self) {
+        ::mediapipe::Status error_status;
+        if (self->GetCombinedErrors(&error_status) && !error_status.ok()) {
+          return error_status.ToString();
+        }
+        return std::string();
+      },
+      R"doc(Combines error messages as a single std::string.
+
+  Examples:
+    if graph.has_error():
+      print(graph.get_combined_error_message())
+
+)doc");
+
+  // TODO: Support passing a single-argument lambda for convenience.
+  calculator_graph.def(
+      "observe_output_stream",
+      [](CalculatorGraph* self, const std::string& stream_name,
+         pybind11::function callback_fn) {
+        RaisePyErrorIfNotOk(self->ObserveOutputStream(
+            stream_name, [callback_fn, stream_name](const Packet& packet) {
+              callback_fn(stream_name, packet);
+              return mediapipe::OkStatus();
+            }));
+      },
+      R"doc(Observe the named output stream.
+
+  callback_fn will be invoked on every packet emitted by the output stream.
+  This method can only be called before start_run().
+
+  Args:
+    stream_name: The name of the output stream.
+    callback_fn: The callback function to invoke on every packet emitted by the
+      output stream.
+
+  Raises:
+    RuntimeError: If the calculator graph isn't initialized or the stream
+      doesn't exist.
+
+  Examples:
+    out = []
+    graph = mp.CalculatorGraph(graph_config=graph_config)
+    graph.observe_output_stream('out',
+                                lambda stream_name, packet: out.append(packet))
+
+)doc");
+
+  calculator_graph.def(
+      "close",
+      [](CalculatorGraph* self) {
+        RaisePyErrorIfNotOk(self->CloseAllPacketSources());
+        RaisePyErrorIfNotOk(self->WaitUntilDone());
+      },
+      R"doc(Close all the input sources and shutdown the graph.)doc");
+
+  calculator_graph.def(
+      "get_output_side_packet",
+      [](CalculatorGraph* self, const std::string& packet_name) {
+        auto status_or_packet = self->GetOutputSidePacket(packet_name);
+        RaisePyErrorIfNotOk(status_or_packet.status());
+        return status_or_packet.ValueOrDie();
+      },
+      R"doc(Get output side packet by name after the graph is done.
+
+  Args:
+    stream: The name of the outnput stream.
+
+  Raises:
+    RuntimeError: If the graph is still running or the output side packet is not
+      found or empty.
+
+  Examples:
+    graph = mp.CalculatorGraph(graph_config=graph_config)
+    graph.start_run()
+    graph.close()
+    output_side_packet = graph.get_output_side_packet('packet_name')
+
+)doc",
+      py::return_value_policy::move);
+}
+
+}  // namespace python
+}  // namespace mediapipe

mediapipe/python/pybind/calculator_graph.h

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_CALCULATOR_GRAPH_H_
+#define MEDIAPIPE_PYTHON_PYBIND_CALCULATOR_GRAPH_H_
+
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+void CalculatorGraphSubmodule(pybind11::module* module);
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_CALCULATOR_GRAPH_H_

mediapipe/python/pybind/image_frame.cc

@@ -0,0 +1,326 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/python/pybind/image_frame_util.h"
+#include "mediapipe/python/pybind/util.h"
+#include "pybind11/stl.h"
+
+namespace mediapipe {
+namespace python {
+namespace {
+
+template <typename T>
+py::array GenerateContiguousDataArray(const ImageFrame& image_frame,
+                                      const py::object& py_object) {
+  std::vector<int> shape{image_frame.Height(), image_frame.Width()};
+  if (image_frame.NumberOfChannels() > 1) {
+    shape.push_back(image_frame.NumberOfChannels());
+  }
+  py::array_t<T, py::array::c_style> contiguous_data;
+  if (image_frame.IsContiguous()) {
+    // TODO: Create contiguous_data without copying ata.
+    // It's possible to achieve this with the help of py::capsule.
+    // Reference: https://github.com/pybind/pybind11/issues/1042,
+    contiguous_data = py::array_t<T, py::array::c_style>(
+        shape, reinterpret_cast<const T*>(image_frame.PixelData()));
+  } else {
+    auto contiguous_data_copy =
+        absl::make_unique<T[]>(image_frame.Width() * image_frame.Height() *
+                               image_frame.NumberOfChannels());
+    image_frame.CopyToBuffer(contiguous_data_copy.get(),
+                             image_frame.PixelDataSizeStoredContiguously());
+    auto capsule = py::capsule(contiguous_data_copy.get(), [](void* data) {
+      if (data) {
+        delete[] reinterpret_cast<T*>(data);
+      }
+    });
+    contiguous_data = py::array_t<T, py::array::c_style>(
+        shape, contiguous_data_copy.release(), capsule);
+  }
+
+  // In both cases, the underlying data is not writable in Python.
+  py::detail::array_proxy(contiguous_data.ptr())->flags &=
+      ~py::detail::npy_api::NPY_ARRAY_WRITEABLE_;
+  return contiguous_data;
+}
+
+py::array GetContiguousDataAttr(const ImageFrame& image_frame,
+                                const py::object& py_object) {
+  py::object get_data_attr =
+      py::getattr(py_object, "__contiguous_data", py::none());
+  if (image_frame.IsEmpty()) {
+    throw RaisePyError(PyExc_RuntimeError, "ImageFrame is unallocated.");
+  }
+  // If __contiguous_data attr already stores data, return the cached results.
+  if (!get_data_attr.is_none()) {
+    return get_data_attr.cast<py::array>();
+  }
+  switch (image_frame.ChannelSize()) {
+    case sizeof(uint8):
+      py_object.attr("__contiguous_data") =
+          GenerateContiguousDataArray<uint8>(image_frame, py_object);
+      break;
+    case sizeof(uint16):
+      py_object.attr("__contiguous_data") =
+          GenerateContiguousDataArray<uint16>(image_frame, py_object);
+      break;
+    case sizeof(float):
+      py_object.attr("__contiguous_data") =
+          GenerateContiguousDataArray<float>(image_frame, py_object);
+      break;
+    default:
+      throw RaisePyError(PyExc_RuntimeError,
+                         "Unsupported image frame channel size. Data is not "
+                         "uint8, uint16, or float?");
+  }
+  return py_object.attr("__contiguous_data").cast<py::array>();
+}
+
+template <typename T>
+py::object GetValue(const ImageFrame& image_frame, const std::vector<int>& pos,
+                    const py::object& py_object) {
+  py::array_t<T, py::array::c_style> output_array =
+      GetContiguousDataAttr(image_frame, py_object);
+  if (pos.size() == 2) {
+    return py::cast(static_cast<T>(output_array.at(pos[0], pos[1])));
+  } else if (pos.size() == 3) {
+    return py::cast(static_cast<T>(output_array.at(pos[0], pos[1], pos[2])));
+  }
+  return py::none();
+}
+
+}  // namespace
+
+namespace py = pybind11;
+
+void ImageFrameSubmodule(pybind11::module* module) {
+  py::module m =
+      module->def_submodule("image_frame", "MediaPipe image frame module");
+
+  py::options options;
+  options.disable_function_signatures();
+
+  // ImageFormat
+  py::enum_<mediapipe::ImageFormat::Format> image_format(
+      m, "ImageFormat",
+      R"doc(An enum describing supported raw image formats.
+
+  SRGB: sRGB, interleaved: one byte for R, then one byte for G, then one byte
+    for B for each pixel.
+
+  SRGBA: sRGBA, interleaved: one byte for R, one byte for G, one byte for B, one
+    byte for alpha or unused.
+
+  SBGRA: sBGRA, interleaved: one byte for B, one byte for G, one byte for R, one
+    byte for alpha or unused.
+
+  GRAY8: Grayscale, one byte per pixel.
+
+  GRAY16: Grayscale, one uint16 per pixel.
+
+  SRGB48: sRGB, interleaved, each component is a uint16.
+
+  SRGBA64: sRGBA, interleaved, each component is a uint16.
+
+  VEC32F1: One float per pixel.
+
+  VEC32F2: Two floats per pixel.
+)doc");
+
+  image_format.value("SRGB", mediapipe::ImageFormat::SRGB)
+      .value("SRGBA", mediapipe::ImageFormat::SRGBA)
+      .value("SBGRA", mediapipe::ImageFormat::SBGRA)
+      .value("GRAY8", mediapipe::ImageFormat::GRAY8)
+      .value("GRAY16", mediapipe::ImageFormat::GRAY16)
+      .value("SRGB48", mediapipe::ImageFormat::SRGB48)
+      .value("SRGBA64", mediapipe::ImageFormat::SRGBA64)
+      .value("VEC32F1", mediapipe::ImageFormat::VEC32F1)
+      .value("VEC32F2", mediapipe::ImageFormat::VEC32F2)
+      .export_values();
+
+  // ImageFrame
+  py::class_<ImageFrame> image_frame(
+      m, "ImageFrame",
+      R"doc(A container for storing an image or a video frame, in one of several formats.
+
+  Formats supported by ImageFrame are listed in the ImageFormat enum.
+  Pixels are encoded row-major in an interleaved fashion. ImageFrame supports
+  uint8, uint16, and float as its data types.
+
+  ImageFrame can be created by copying the data from a numpy ndarray that stores
+  the pixel data continuously. An ImageFrame may realign the input data on its
+  default alignment boundary during creation. The data in an ImageFrame will
+  become immutable after creation.
+
+  Creation examples:
+    import cv2
+    cv_mat = cv2.imread(input_file)[:, :, ::-1]
+    rgb_frame = mp.ImageFrame(format=ImageFormat.SRGB, data=cv_mat)
+    gray_frame = mp.ImageFrame(
+        format=ImageFormat.GRAY, data=cv2.cvtColor(cv_mat, cv2.COLOR_RGB2GRAY))
+
+    from PIL import Image
+    pil_img = Image.new('RGB', (60, 30), color = 'red')
+    image_frame = mp.ImageFrame(
+        format=mp.ImageFormat.SRGB, data=np.asarray(pil_img))
+
+  The pixel data in an ImageFrame can be retrieved as a numpy ndarray by calling
+  `ImageFrame.numpy_view()`. The returned numpy ndarray is a reference to the
+  internal data and itself is unwritable. If the callers want to modify the
+  numpy ndarray, it's required to obtain a copy of it.
+
+  Pixel data retrieval examples:
+    for channel in range(num_channel):
+      for col in range(width):
+        for row in range(height):
+          print(image_frame[row, col, channel])
+
+    output_ndarray = image_frame.numpy_view()
+    print(output_ndarray[0, 0, 0])
+    copied_ndarray = np.copy(output_ndarray)
+    copied_ndarray[0,0,0] = 0
+  )doc",
+      py::dynamic_attr());
+
+  image_frame
+      .def(
+          py::init([](mediapipe::ImageFormat::Format format,
+                      const py::array_t<uint8, py::array::c_style>& data) {
+            if (format != mediapipe::ImageFormat::GRAY8 &&
+                format != mediapipe::ImageFormat::SRGB &&
+                format != mediapipe::ImageFormat::SRGBA) {
+              throw RaisePyError(PyExc_RuntimeError,
+                                 "uint8 image data should be one of the GRAY8, "
+                                 "SRGB, and SRGBA MediaPipe image formats.");
+            }
+            return CreateImageFrame<uint8>(format, data);
+          }),
+          R"doc(For uint8 data type, valid ImageFormat are GRAY8, SGRB, and SRGBA.)doc",
+          py::arg("image_format"), py::arg("data").noconvert())
+      .def(
+          py::init([](mediapipe::ImageFormat::Format format,
+                      const py::array_t<uint16, py::array::c_style>& data) {
+            if (format != mediapipe::ImageFormat::GRAY16 &&
+                format != mediapipe::ImageFormat::SRGB48 &&
+                format != mediapipe::ImageFormat::SRGBA64) {
+              throw RaisePyError(
+                  PyExc_RuntimeError,
+                  "uint16 image data should be one of the GRAY16, "
+                  "SRGB48, and SRGBA64 MediaPipe image formats.");
+            }
+            return CreateImageFrame<uint16>(format, data);
+          }),
+          R"doc(For uint16 data type, valid ImageFormat are GRAY16, SRGB48, and SRGBA64.)doc",
+          py::arg("image_format"), py::arg("data").noconvert())
+      .def(
+          py::init([](mediapipe::ImageFormat::Format format,
+                      const py::array_t<float, py::array::c_style>& data) {
+            if (format != mediapipe::ImageFormat::VEC32F1 &&
+                format != mediapipe::ImageFormat::VEC32F2) {
+              throw RaisePyError(
+                  PyExc_RuntimeError,
+                  "float image data should be either VEC32F1 or VEC32F2 "
+                  "MediaPipe image formats.");
+            }
+            return CreateImageFrame<float>(format, data);
+          }),
+          R"doc(For float data type, valid ImageFormat are VEC32F1 and VEC32F2.)doc",
+          py::arg("image_format"), py::arg("data").noconvert());
+
+  image_frame.def(
+      "numpy_view",
+      [](ImageFrame& self) {
+        py::object py_object =
+            py::cast(self, py::return_value_policy::reference);
+        return GetContiguousDataAttr(self, py_object);
+      },
+      R"doc(Return the image frame pixel data as an unwritable numpy ndarray.
+
+  Realign the pixel data to be stored contiguously and return a reference to the
+  unwritable numpy ndarray. If the callers want to modify the numpy array data,
+  it's required to obtain a copy of the ndarray.
+
+  Returns:
+    An unwritable numpy ndarray.
+
+  Examples:
+    output_ndarray = image_frame.numpy_view()
+    copied_ndarray = np.copy(output_ndarray)
+    copied_ndarray[0,0,0] = 0
+)doc");
+
+  image_frame.def(
+      "__getitem__",
+      [](ImageFrame& self, const std::vector<int>& pos) {
+        if (pos.size() != 3 &&
+            !(pos.size() == 2 && self.NumberOfChannels() == 1)) {
+          throw RaisePyError(
+              PyExc_IndexError,
+              absl::StrCat("Invalid index dimension: ", pos.size()).c_str());
+        }
+        py::object py_object =
+            py::cast(self, py::return_value_policy::reference);
+        switch (self.ByteDepth()) {
+          case 1:
+            return GetValue<uint8>(self, pos, py_object);
+          case 2:
+            return GetValue<uint16>(self, pos, py_object);
+          case 4:
+            return GetValue<float>(self, pos, py_object);
+          default:
+            return py::object();
+        }
+      },
+      R"doc(Use the indexer operators to access pixel data.
+
+  Raises:
+    IndexError: If the index is invalid or out of bounds.
+
+  Examples:
+    for channel in range(num_channel):
+      for col in range(width):
+        for row in range(height):
+          print(image_frame[row, col, channel])
+
+)doc");
+
+  image_frame
+      .def(
+          "is_contiguous", &ImageFrame::IsContiguous,
+          R"doc(Return True if the pixel data is stored contiguously (without any alignment padding areas).)doc")
+      .def("is_empty", &ImageFrame::IsEmpty,
+           R"doc(Return True if the pixel data is unallocated.)doc")
+      .def(
+          "is_aligned", &ImageFrame::IsAligned,
+          R"doc(Return True if each row of the data is aligned to alignment boundary, which must be 1 or a power of 2.
+
+  Args:
+    alignment_boundary: An integer.
+
+  Returns:
+    A boolean.
+
+  Examples:
+    image_frame.is_aligned(16)
+)doc");
+
+  image_frame.def_property_readonly("width", &ImageFrame::Width)
+      .def_property_readonly("height", &ImageFrame::Height)
+      .def_property_readonly("channels", &ImageFrame::NumberOfChannels)
+      .def_property_readonly("byte_depth", &ImageFrame::ByteDepth)
+      .def_property_readonly("image_format", &ImageFrame::Format);
+}
+
+}  // namespace python
+}  // namespace mediapipe

mediapipe/python/pybind/image_frame.h

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_IMAGE_FRAME_H_
+#define MEDIAPIPE_PYTHON_PYBIND_IMAGE_FRAME_H_
+
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+void ImageFrameSubmodule(pybind11::module* module);
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_IMAGE_FRAME_H_

mediapipe/python/pybind/image_frame_util.h

@@ -0,0 +1,61 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_IMAGE_FRAME_UTIL_H_
+#define MEDIAPIPE_PYTHON_PYBIND_IMAGE_FRAME_UTIL_H_
+
+#include "absl/memory/memory.h"
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/logging.h"
+#include "pybind11/numpy.h"
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+namespace py = pybind11;
+
+// TODO: Implement the reference mode of image frame creation, which
+// takes a reference to the external data rather than copying it over.
+// A possible solution is to have a custom PixelDataDeleter:
+// The refcount of the numpy array will be increased when the image frame is
+// created by taking a reference to the external numpy array data. Then, the
+// custom PixelDataDeleter will decrease the refcount when the image frame gets
+// destroyed and let Python GC does its job.
+template <typename T>
+std::unique_ptr<ImageFrame> CreateImageFrame(
+    mediapipe::ImageFormat::Format format,
+    const py::array_t<T, py::array::c_style>& data) {
+  int rows = data.shape()[0];
+  int cols = data.shape()[1];
+  int width_step = ImageFrame::NumberOfChannelsForFormat(format) *
+                   ImageFrame::ByteDepthForFormat(format) * cols;
+  auto image_frame = absl::make_unique<ImageFrame>(
+      format, /*width=*/cols, /*height=*/rows, width_step,
+      static_cast<uint8*>(data.request().ptr),
+      ImageFrame::PixelDataDeleter::kNone);
+  auto image_frame_copy = absl::make_unique<ImageFrame>();
+  // Set alignment_boundary to kGlDefaultAlignmentBoundary so that both
+  // GPU and CPU can process it.
+  image_frame_copy->CopyFrom(*image_frame,
+                             ImageFrame::kGlDefaultAlignmentBoundary);
+  return image_frame_copy;
+}
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_IMAGE_FRAME_UTIL_H_

mediapipe/python/pybind/matrix.cc

@@ -0,0 +1,36 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/python/pybind/matrix.h"
+
+#include "mediapipe/framework/formats/matrix.h"
+#include "pybind11/numpy.h"
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+namespace py = pybind11;
+
+void MatrixSubmodule(pybind11::module* module) {
+  py::module m = module->def_submodule("matrix", "MediaPipe matrix module.");
+
+  py::class_<mediapipe::Matrix>(m, "Matrix")
+      .def(py::init(
+          // Pass by reference.
+          [](const Eigen::Ref<const Eigen::MatrixXf>& m) { return m; }));
+}
+
+}  // namespace python
+}  // namespace mediapipe

mediapipe/python/pybind/matrix.h

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_MATRIX_H_
+#define MEDIAPIPE_PYTHON_PYBIND_MATRIX_H_
+
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+void MatrixSubmodule(pybind11::module* module);
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_MATRIX_H_

mediapipe/python/pybind/packet.cc

@@ -0,0 +1,72 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/python/pybind/packet.h"
+
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/python/pybind/util.h"
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+namespace py = pybind11;
+
+void PacketSubmodule(pybind11::module* module) {
+  py::module m = module->def_submodule("packet", "MediaPipe packet module.");
+
+  py::class_<Packet> packet(
+      m, "Packet",
+      R"doc(The basic data flow unit of MediaPipe. A generic container class which can hold data of any type.
+
+  A packet consists of a numeric timestamp and a shared pointer to an immutable
+  payload. The payload can be of any C++ type (See packet_creator module for
+  the list of the Python types that are supported). The payload's type is also
+  referred to as the type of the packet. Packets are value classes and can be
+  copied and moved cheaply. Each copy shares ownership of the payload, with be
+  copied reference-counting semantics. Each copy has its own timestamp.
+
+  The preferred method of creating a Packet is to invoke the methods in the
+  "packet_creator" module. Packet contents can be retrieved by the methods in
+  the "packet_getter" module.
+)doc");
+
+  packet.def(py::init(),
+             R"doc(Create an empty Packet, for which is_empty() is True and
+  timestamp() is Timestamp.unset. Calling packet getter methods on this Packet leads to runtime error.)doc");
+
+  packet.def(
+      "is_empty", &Packet::IsEmpty,
+      R"doc(Return true iff the Packet has been created using the default constructor Packet(), or is a copy of such a Packet.)doc");
+
+  packet.def(py::init<Packet const&>())
+      .def("at", [](Packet* self,
+                    int64 ts_value) { return self->At(Timestamp(ts_value)); })
+      .def("at", [](Packet* self, Timestamp ts) { return self->At(ts); })
+      .def_property(
+          "timestamp", &Packet::Timestamp,
+          [](Packet* p, int64 ts_value) { *p = p->At(Timestamp(ts_value)); })
+      .def("__repr__", [](const Packet& self) {
+        return absl::StrCat(
+            "<mediapipe.Packet with timestamp: ",
+            TimestampValueString(self.Timestamp()),
+            self.IsEmpty()
+                ? " and no data>"
+                : absl::StrCat(" and C++ type: ", self.DebugTypeName(), ">"));
+      });
+}
+
+}  // namespace python
+}  // namespace mediapipe

mediapipe/python/pybind/packet.h

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_PACKET_H_
+#define MEDIAPIPE_PYTHON_PYBIND_PACKET_H_
+
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+void PacketSubmodule(pybind11::module* module);
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_PACKET_H_

mediapipe/python/pybind/packet_creator.cc

@@ -0,0 +1,645 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/python/pybind/packet_creator.h"
+
+#include "absl/memory/memory.h"
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/python/pybind/image_frame_util.h"
+#include "mediapipe/python/pybind/util.h"
+#include "pybind11/eigen.h"
+#include "pybind11/pybind11.h"
+#include "pybind11/stl.h"
+
+namespace mediapipe {
+namespace python {
+namespace {
+
+Packet CreateImageFramePacket(mediapipe::ImageFormat::Format format,
+                              const py::array& data) {
+  if (format == mediapipe::ImageFormat::SRGB ||
+      format == mediapipe::ImageFormat::SRGBA ||
+      format == mediapipe::ImageFormat::GRAY8) {
+    return Adopt(CreateImageFrame<uint8>(format, data).release());
+  } else if (format == mediapipe::ImageFormat::GRAY16 ||
+             format == mediapipe::ImageFormat::SRGB48 ||
+             format == mediapipe::ImageFormat::SRGBA64) {
+    return Adopt(CreateImageFrame<uint16>(format, data).release());
+  } else if (format == mediapipe::ImageFormat::VEC32F1 ||
+             format == mediapipe::ImageFormat::VEC32F2) {
+    return Adopt(CreateImageFrame<float>(format, data).release());
+  }
+  throw RaisePyError(PyExc_RuntimeError,
+                     absl::StrCat("Unsupported ImageFormat: ", format).c_str());
+  return Packet();
+}
+
+}  // namespace
+
+namespace py = pybind11;
+
+void PublicPacketCreators(pybind11::module* m) {
+  m->def(
+      "create_string",
+      [](const std::string& data) { return MakePacket<std::string>(data); },
+      R"doc(Create a MediaPipe std::string Packet from a str.
+
+  Args:
+    data: A str.
+
+  Returns:
+    A MediaPipe std::string Packet.
+
+  Raises:
+    TypeError: If the input is not a str.
+
+  Examples:
+    packet = mp.packet_creator.create_string('abc')
+    data = mp.packet_getter.get_string(packet)
+)doc",
+      py::return_value_policy::move);
+
+  m->def(
+      "create_string",
+      [](const py::bytes& data) { return MakePacket<std::string>(data); },
+      R"doc(Create a MediaPipe std::string Packet from a bytes object.
+
+  Args:
+    data: A bytes object.
+
+  Returns:
+    A MediaPipe std::string Packet.
+
+  Raises:
+    TypeError: If the input is not a bytes object.
+
+  Examples:
+    packet = mp.packet_creator.create_string(b'\xd0\xd0\xd0')
+    data = mp.packet_getter.get_bytes(packet)
+)doc",
+      py::return_value_policy::move);
+
+  m->def(
+      "create_bool", [](bool data) { return MakePacket<bool>(data); },
+      R"doc(Create a MediaPipe bool Packet from a boolean object.
+
+  Args:
+    data: A boolean object.
+
+  Returns:
+    A MediaPipe bool Packet.
+
+  Raises:
+    TypeError: If the input is not a boolean object.
+
+  Examples:
+    packet = mp.packet_creator.create_bool(True)
+    data = mp.packet_getter.get_bool(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_int",
+      [](int64 data) {
+        RaisePyErrorIfOverflow(data, INT_MIN, INT_MAX);
+        return MakePacket<int>(data);
+      },
+      R"doc(Create a MediaPipe int Packet from an integer.
+
+  Args:
+    data: An integer or a np.intc.
+
+  Returns:
+    A MediaPipe int Packet.
+
+  Raises:
+    OverflowError: If the input integer overflows.
+    TypeError: If the input is not an integer.
+
+  Examples:
+    packet = mp.packet_creator.create_int(0)
+    data = mp.packet_getter.get_int(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_int8",
+      [](int64 data) {
+        RaisePyErrorIfOverflow(data, INT8_MIN, INT8_MAX);
+        return MakePacket<int8>(data);
+      },
+      R"doc(Create a MediaPipe int8 Packet from an integer.
+
+  Args:
+    data: An integer or a np.int8.
+
+  Returns:
+    A MediaPipe int8 Packet.
+
+  Raises:
+    OverflowError: If the input integer overflows.
+    TypeError: If the input is neither an integer nor a np.int8.
+
+  Examples:
+    packet = mp.packet_creator.create_int8(2**7 - 1)
+    data = mp.packet_getter.get_int(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_int16",
+      [](int64 data) {
+        RaisePyErrorIfOverflow(data, INT16_MIN, INT16_MAX);
+        return MakePacket<int16>(data);
+      },
+      R"doc(Create a MediaPipe int16 Packet from an integer.
+
+  Args:
+    data: An integer or a np.int16.
+
+  Returns:
+    A MediaPipe int16 Packet.
+
+  Raises:
+    OverflowError: If the input integer overflows.
+    TypeError: If the input is neither an integer nor a np.int16.
+
+  Examples:
+    packet = mp.packet_creator.create_int16(2**15 - 1)
+    data = mp.packet_getter.get_int(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_int32",
+      [](int64 data) {
+        RaisePyErrorIfOverflow(data, INT32_MIN, INT32_MAX);
+        return MakePacket<int32>(data);
+      },
+      R"doc(Create a MediaPipe int32 Packet from an integer.
+
+  Args:
+    data: An integer or a np.int32.
+
+  Returns:
+    A MediaPipe int32 Packet.
+
+  Raises:
+    OverflowError: If the input integer overflows.
+    TypeError: If the input is neither an integer nor a np.int32.
+
+  Examples:
+    packet = mp.packet_creator.create_int32(2**31 - 1)
+    data = mp.packet_getter.get_int(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_int64", [](int64 data) { return MakePacket<int64>(data); },
+      R"doc(Create a MediaPipe int64 Packet from an integer.
+
+  Args:
+    data: An integer or a np.int64.
+
+  Returns:
+    A MediaPipe int64 Packet.
+
+  Raises:
+    TypeError: If the input is neither an integer nor a np.int64.
+
+  Examples:
+    packet = mp.packet_creator.create_int64(2**63 - 1)
+    data = mp.packet_getter.get_int(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_uint8",
+      [](int64 data) {
+        RaisePyErrorIfOverflow(data, 0, UINT8_MAX);
+        return MakePacket<uint8>(data);
+      },
+      R"doc(Create a MediaPipe uint8 Packet from an integer.
+
+  Args:
+    data: An integer or a np.uint8.
+
+  Returns:
+    A MediaPipe uint8 Packet.
+
+  Raises:
+    OverflowError: If the input integer overflows.
+    TypeError: If the input is neither an integer nor a np.uint8.
+
+  Examples:
+    packet = mp.packet_creator.create_uint8(2**8 - 1)
+    data = mp.packet_getter.get_uint(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_uint16",
+      [](int64 data) {
+        RaisePyErrorIfOverflow(data, 0, UINT16_MAX);
+        return MakePacket<uint16>(data);
+      },
+      R"doc(Create a MediaPipe uint16 Packet from an integer.
+
+  Args:
+    data: An integer or a np.uint16.
+
+  Returns:
+    A MediaPipe uint16 Packet.
+
+  Raises:
+    OverflowError: If the input integer overflows.
+    TypeError: If the input is neither an integer nor a np.uint16.
+
+  Examples:
+    packet = mp.packet_creator.create_uint16(2**16 - 1)
+    data = mp.packet_getter.get_uint(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_uint32",
+      [](int64 data) {
+        RaisePyErrorIfOverflow(data, 0, UINT32_MAX);
+        return MakePacket<uint32>(data);
+      },
+      R"doc(Create a MediaPipe uint32 Packet from an integer.
+
+  Args:
+    data: An integer or a np.uint32.
+
+  Returns:
+    A MediaPipe uint32 Packet.
+
+  Raises:
+    OverflowError: If the input integer overflows.
+    TypeError: If the input is neither an integer nor a np.uint32.
+
+  Examples:
+    packet = mp.packet_creator.create_uint32(2**32 - 1)
+    data = mp.packet_getter.get_uint(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_uint64", [](uint64 data) { return MakePacket<uint64>(data); },
+      R"doc(Create a MediaPipe uint64 Packet from an integer.
+
+  Args:
+    data: An integer or a np.uint64.
+
+  Returns:
+    A MediaPipe uint64 Packet.
+
+  Raises:
+    TypeError: If the input is neither an integer nor a np.uint64.
+
+  Examples:
+    packet = mp.packet_creator.create_uint64(2**64 - 1)
+    data = mp.packet_getter.get_uint(packet)
+)doc",
+      // py::arg().noconvert() won't allow this to accept np.uint64 data type.
+      py::arg(), py::return_value_policy::move);
+
+  m->def(
+      "create_float", [](float data) { return MakePacket<float>(data); },
+      R"doc(Create a MediaPipe float Packet from a float.
+
+  Args:
+    data: A float or a np.float.
+
+  Returns:
+    A MediaPipe float Packet.
+
+  Raises:
+    TypeError: If the input is neither a float nor a np.float.
+
+  Examples:
+    packet = mp.packet_creator.create_float(0.1)
+    data = mp.packet_getter.get_float(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_double", [](double data) { return MakePacket<double>(data); },
+      R"doc(Create a MediaPipe double Packet from a float.
+
+  Args:
+    data: A float or a np.double.
+
+  Returns:
+    A MediaPipe double Packet.
+
+  Raises:
+    TypeError: If the input is neither a float nore a np.double.
+
+  Examples:
+    packet = mp.packet_creator.create_double(0.1)
+    data = mp.packet_getter.get_float(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_int_array",
+      [](const std::vector<int>& data) {
+        int* ints = new int[data.size()];
+        std::copy(data.begin(), data.end(), ints);
+        return Adopt(reinterpret_cast<int(*)[]>(ints));
+      },
+      R"doc(Create a MediaPipe int array Packet from a list of integers.
+
+  Args:
+    data: A list of integers.
+
+  Returns:
+    A MediaPipe int array Packet.
+
+  Raises:
+    TypeError: If the input is not a list of integers.
+
+  Examples:
+    packet = mp.packet_creator.create_int_array([1, 2, 3])
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_float_array",
+      [](const std::vector<float>& data) {
+        float* floats = new float[data.size()];
+        std::copy(data.begin(), data.end(), floats);
+        return Adopt(reinterpret_cast<float(*)[]>(floats));
+      },
+      R"doc(Create a MediaPipe float array Packet from a list of floats.
+
+  Args:
+    data: A list of floats.
+
+  Returns:
+    A MediaPipe float array Packet.
+
+  Raises:
+    TypeError: If the input is not a list of floats.
+
+  Examples:
+    packet = mp.packet_creator.create_float_array([0.1, 0.2, 0.3])
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_int_vector",
+      [](const std::vector<int>& data) {
+        return MakePacket<std::vector<int>>(data);
+      },
+      R"doc(Create a MediaPipe int vector Packet from a list of integers.
+
+  Args:
+    data: A list of integers.
+
+  Returns:
+    A MediaPipe int vector Packet.
+
+  Raises:
+    TypeError: If the input is not a list of integers.
+
+  Examples:
+    packet = mp.packet_creator.create_int_vector([1, 2, 3])
+    data = mp.packet_getter.get_int_vector(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_float_vector",
+      [](const std::vector<float>& data) {
+        return MakePacket<std::vector<float>>(data);
+      },
+      R"doc(Create a MediaPipe float vector Packet from a list of floats.
+
+  Args:
+    data: A list of floats
+
+  Returns:
+    A MediaPipe float vector Packet.
+
+  Raises:
+    TypeError: If the input is not a list of floats.
+
+  Examples:
+    packet = mp.packet_creator.create_float_vector([0.1, 0.2, 0.3])
+    data = mp.packet_getter.get_float_list(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_string_vector",
+      [](const std::vector<std::string>& data) {
+        return MakePacket<std::vector<std::string>>(data);
+      },
+      R"doc(Create a MediaPipe std::string vector Packet from a list of str.
+
+  Args:
+    data: A list of str.
+
+  Returns:
+    A MediaPipe std::string vector Packet.
+
+  Raises:
+    TypeError: If the input is not a list of str.
+
+  Examples:
+    packet = mp.packet_creator.create_string_vector(['a', 'b', 'c'])
+    data = mp.packet_getter.get_str_list(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_packet_vector",
+      [](const std::vector<Packet>& data) {
+        return MakePacket<std::vector<Packet>>(data);
+      },
+      R"doc(Create a MediaPipe Packet holds a vector of packets.
+
+  Args:
+    data: A list of packets.
+
+  Returns:
+    A MediaPipe Packet holds a vector of packets.
+
+  Raises:
+    TypeError: If the input is not a list of packets.
+
+  Examples:
+    packet = mp.packet_creator.create_packet_vector([
+        mp.packet_creator.create_float(0.1),
+        mp.packet_creator.create_int(1),
+        mp.packet_creator.create_string('1')
+    ])
+   data = mp.packet_getter.get_packet_vector(packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_string_to_packet_map",
+      [](const std::map<std::string, Packet>& data) {
+        return MakePacket<std::map<std::string, Packet>>(data);
+      },
+      R"doc(Create a MediaPipe std::string to packet map Packet from a dictionary.
+
+  Args:
+    data: A dictionary that has (str, Packet) pairs.
+
+  Returns:
+    A MediaPipe Packet holds std::map<std::string, Packet>.
+
+  Raises:
+    TypeError: If the input is not a dictionary from str to packet.
+
+  Examples:
+    dict_packet = mp.packet_creator.create_string_to_packet_map({
+        'float': mp.packet_creator.create_float(0.1),
+        'int': mp.packet_creator.create_int(1),
+        'std::string': mp.packet_creator.create_string('1')
+    data = mp.packet_getter.get_str_to_packet_dict(dict_packet)
+)doc",
+      py::arg().noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "create_matrix",
+      // Eigen Map class
+      // (https://eigen.tuxfamily.org/dox/group__TutorialMapClass.html) is the
+      // way to reuse the external memory as an Eigen type. However, when
+      // creating an Eigen::MatrixXf from an Eigen Map object, the data copy
+      // still happens. We can  make a packet of an Eigen Map type for reusing
+      // external memory. However,the packet data type is no longer
+      // Eigen::MatrixXf.
+      // TODO: Should take "const Eigen::Ref<const Eigen::MatrixXf>&"
+      // as the input argument. Investigate why bazel non-optimized mode
+      // triggers a memory allocation bug in Eigen::internal::aligned_free().
+      [](const Eigen::MatrixXf& matrix) {
+        // MakePacket copies the data.
+        return MakePacket<Matrix>(matrix);
+      },
+      R"doc(Create a MediaPipe Matrix Packet from a 2d numpy float ndarray.
+
+  The method copies data from the input MatrixXf and the returned packet owns
+  a MatrixXf object.
+
+  Args:
+    matrix: A 2d numpy float ndarray.
+
+  Returns:
+    A MediaPipe Matrix Packet.
+
+  Raises:
+    TypeError: If the input is not a 2d numpy float ndarray.
+
+  Examples:
+    packet = mp.packet_creator.create_matrix(
+        np.array([[.1, .2, .3], [.4, .5, .6]])
+    matrix = mp.packet_getter.get_matrix(packet)
+)doc",
+      py::return_value_policy::move);
+}
+
+void InternalPacketCreators(pybind11::module* m) {
+  m->def(
+      "_create_image_frame_with_copy",
+      [](mediapipe::ImageFormat::Format format, const py::array& data) {
+        return CreateImageFramePacket(format, data);
+      },
+      py::arg("format"), py::arg("data").noconvert(),
+      py::return_value_policy::move);
+
+  m->def(
+      "_create_image_frame_with_reference",
+      [](mediapipe::ImageFormat::Format format, const py::array& data) {
+        throw RaisePyError(
+            PyExc_NotImplementedError,
+            "Creating image frame packet with reference is not supproted yet.");
+      },
+      py::arg("format"), py::arg("data").noconvert(),
+      py::return_value_policy::move);
+
+  m->def(
+      "_create_image_frame_with_copy",
+      [](ImageFrame& image_frame) {
+        auto image_frame_copy = absl::make_unique<ImageFrame>();
+        // Set alignment_boundary to kGlDefaultAlignmentBoundary so that
+        // both GPU and CPU can process it.
+        image_frame_copy->CopyFrom(image_frame,
+                                   ImageFrame::kGlDefaultAlignmentBoundary);
+        return Adopt(image_frame_copy.release());
+      },
+      py::arg("image_frame").noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "_create_image_frame_with_reference",
+      [](ImageFrame& image_frame) {
+        throw RaisePyError(
+            PyExc_NotImplementedError,
+            "Creating image frame packet with reference is not supproted yet.");
+      },
+      py::arg("image_frame").noconvert(), py::return_value_policy::move);
+
+  m->def(
+      "_create_proto",
+      [](const std::string& type_name, const py::bytes& serialized_proto) {
+        using packet_internal::HolderBase;
+        mediapipe::StatusOr<std::unique_ptr<HolderBase>> maybe_holder =
+            packet_internal::MessageHolderRegistry::CreateByName(type_name);
+        if (!maybe_holder.ok()) {
+          throw RaisePyError(
+              PyExc_RuntimeError,
+              absl::StrCat("Unregistered proto message type: ", type_name)
+                  .c_str());
+        }
+        // Creates a Packet with the concrete C++ payload type.
+        std::unique_ptr<HolderBase> message_holder =
+            std::move(maybe_holder).ValueOrDie();
+        auto* copy = const_cast<proto_ns::MessageLite*>(
+            message_holder->GetProtoMessageLite());
+        copy->ParseFromString(serialized_proto);
+        return packet_internal::Create(message_holder.release());
+      },
+      py::return_value_policy::move);
+
+  m->def(
+      "_create_proto_vector",
+      [](const std::string& type_name,
+         const std::vector<py::bytes>& serialized_proto_vector) {
+        // TODO: Implement this.
+        throw RaisePyError(PyExc_NotImplementedError,
+                           "Creating a packet from a vector of proto messages "
+                           "is not supproted yet.");
+        return Packet();
+      },
+      py::return_value_policy::move);
+}
+
+void PacketCreatorSubmodule(pybind11::module* module) {
+  py::module m = module->def_submodule(
+      "_packet_creator", "MediaPipe internal packet creator module.");
+  PublicPacketCreators(&m);
+  InternalPacketCreators(&m);
+}
+
+}  // namespace python
+}  // namespace mediapipe

mediapipe/python/pybind/packet_creator.h

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_PACKET_CREATOR_H_
+#define MEDIAPIPE_PYTHON_PYBIND_PACKET_CREATOR_H_
+
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+void PacketCreatorSubmodule(pybind11::module* module);
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_PACKET_CREATOR_H_

mediapipe/python/pybind/packet_getter.cc

@@ -0,0 +1,395 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/python/pybind/packet_getter.h"
+
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/python/pybind/image_frame_util.h"
+#include "mediapipe/python/pybind/util.h"
+#include "pybind11/eigen.h"
+#include "pybind11/pybind11.h"
+#include "pybind11/stl.h"
+
+namespace mediapipe {
+namespace python {
+namespace {
+
+template <typename T>
+const T& GetContent(const Packet& packet) {
+  RaisePyErrorIfNotOk(packet.ValidateAsType<T>());
+  return packet.Get<T>();
+}
+
+}  // namespace
+
+namespace py = pybind11;
+
+void PublicPacketGetters(pybind11::module* m) {
+  m->def("get_str", &GetContent<std::string>,
+         R"doc(Get the content of a MediaPipe std::string Packet as a str.
+
+  Args:
+    packet: A MediaPipe std::string Packet.
+
+  Returns:
+    A str.
+
+  Raises:
+    ValueError: If the Packet doesn't contain std::string data.
+
+  Examples:
+    packet = mp.packet_creator.create_string('abc')
+    data = mp.packet_getter.get_str(packet)
+)doc");
+
+  m->def(
+      "get_bytes",
+      [](const Packet& packet) {
+        return py::bytes(GetContent<std::string>(packet));
+      },
+      R"doc(Get the content of a MediaPipe std::string Packet as a bytes object.
+
+  Args:
+    packet: A MediaPipe std::string Packet.
+
+  Returns:
+    A bytes object.
+
+  Raises:
+    ValueError: If the Packet doesn't contain std::string data.
+
+  Examples:
+    packet = mp.packet_creator.create_string(b'\xd0\xd0\xd0')
+    data = mp.packet_getter.get_bytes(packet)
+)doc");
+
+  m->def("get_bool", &GetContent<bool>,
+         R"doc(Get the content of a MediaPipe bool Packet as a boolean.
+
+  Args:
+    packet: A MediaPipe bool Packet.
+
+  Returns:
+    A boolean.
+
+  Raises:
+    ValueError: If the Packet doesn't contain bool data.
+
+  Examples:
+    packet = mp.packet_creator.create_bool(True)
+    data = mp.packet_getter.get_bool(packet)
+)doc");
+
+  m->def(
+      "get_int",
+      [](const Packet& packet) {
+        if (packet.ValidateAsType<int>().ok()) {
+          return static_cast<int64>(packet.Get<int>());
+        } else if (packet.ValidateAsType<int8>().ok()) {
+          return static_cast<int64>(packet.Get<int8>());
+        } else if (packet.ValidateAsType<int16>().ok()) {
+          return static_cast<int64>(packet.Get<int16>());
+        } else if (packet.ValidateAsType<int32>().ok()) {
+          return static_cast<int64>(packet.Get<int32>());
+        } else if (packet.ValidateAsType<int64>().ok()) {
+          return static_cast<int64>(packet.Get<int64>());
+        }
+        throw RaisePyError(
+            PyExc_ValueError,
+            "Packet doesn't contain int, int8, int16, int32, or int64 data.");
+      },
+      R"doc(Get the content of a MediaPipe int Packet as an integer.
+
+  Args:
+    packet: A MediaPipe Packet that holds int, int8, int16, int32, or int64 data.
+
+  Returns:
+    An integer.
+
+  Raises:
+    ValueError: If the Packet doesn't contain int, int8, int16, int32, or int64 data.
+
+  Examples:
+    packet = mp.packet_creator.create_int(0)
+    data = mp.packet_getter.get_int(packet)
+)doc");
+
+  m->def(
+      "get_uint",
+      [](const Packet& packet) {
+        if (packet.ValidateAsType<uint8>().ok()) {
+          return static_cast<std::uint64_t>(packet.Get<uint8>());
+        } else if (packet.ValidateAsType<uint16>().ok()) {
+          return static_cast<std::uint64_t>(packet.Get<uint16>());
+        } else if (packet.ValidateAsType<uint32>().ok()) {
+          return static_cast<std::uint64_t>(packet.Get<uint32>());
+        } else if (packet.ValidateAsType<uint64>().ok()) {
+          return static_cast<std::uint64_t>(packet.Get<uint64>());
+        }
+        throw RaisePyError(
+            PyExc_ValueError,
+            "Packet doesn't contain uint8, uint16, uint32, or uint64 data.");
+      },
+      R"doc(Get the content of a MediaPipe uint Packet as an integer.
+
+  Args:
+    packet: A MediaPipe Packet that holds uint8, uint16, uint32, or uint64 data.
+
+  Raises:
+    ValueError: If the Packet doesn't contain uint8, uint16, uint32, or uint64 data.
+
+  Returns:
+    An integer.
+
+  Examples:
+    packet = mp.packet_creator.create_uint8(2**8 - 1)
+    data = mp.packet_getter.get_uint(packet)
+)doc");
+
+  m->def(
+      "get_float",
+      [](const Packet& packet) {
+        if (packet.ValidateAsType<float>().ok()) {
+          return packet.Get<float>();
+        } else if (packet.ValidateAsType<double>().ok()) {
+          return static_cast<float>(packet.Get<double>());
+        }
+        throw RaisePyError(PyExc_ValueError,
+                           "Packet doesn't contain float or double data.");
+      },
+      R"doc(Get the content of a MediaPipe float or double Packet as a float.
+
+  Args:
+    packet: A MediaPipe Packet that holds float or double data.
+
+  Raises:
+    ValueError: If the Packet doesn't contain float or double data.
+
+  Returns:
+    A float.
+
+  Examples:
+    packet = mp.packet_creator.create_float(0.1)
+    data = mp.packet_getter.get_float(packet)
+)doc");
+
+  m->def(
+      "get_int_list", &GetContent<std::vector<int>>,
+      R"doc(Get the content of a MediaPipe int vector Packet as an integer list.
+
+  Args:
+    packet: A MediaPipe Packet that holds std:vector<int>.
+
+  Returns:
+    An integer list.
+
+  Raises:
+    ValueError: If the Packet doesn't contain std:vector<int>.
+
+  Examples:
+    packet = mp.packet_creator.create_int_vector([1, 2, 3])
+    data = mp.packet_getter.get_int_list(packet)
+)doc");
+
+  m->def(
+      "get_float_list", &GetContent<std::vector<float>>,
+      R"doc(Get the content of a MediaPipe float vector Packet as a float list.
+
+  Args:
+    packet: A MediaPipe Packet that holds std:vector<float>.
+
+  Returns:
+    A float list.
+
+  Raises:
+    ValueError: If the Packet doesn't contain std:vector<float>.
+
+  Examples:
+    packet = packet_creator.create_float_vector([0.1, 0.2, 0.3])
+    data = packet_getter.get_float_list(packet)
+)doc");
+
+  m->def(
+      "get_str_list", &GetContent<std::vector<std::string>>,
+      R"doc(Get the content of a MediaPipe std::string vector Packet as a str list.
+
+  Args:
+    packet: A MediaPipe Packet that holds std:vector<std::string>.
+
+  Returns:
+    A str list.
+
+  Raises:
+    ValueError: If the Packet doesn't contain std:vector<std::string>.
+
+  Examples:
+    packet = mp.packet_creator.create_string_vector(['a', 'b', 'c'])
+    data = mp.packet_getter.get_str_list(packet)
+)doc");
+
+  m->def(
+      "get_packet_list", &GetContent<std::vector<Packet>>,
+      R"doc(Get the content of a MediaPipe Packet of Packet vector as a Packet list.
+
+  Args:
+    packet: A MediaPipe Packet that holds std:vector<Packet>.
+
+  Returns:
+    A Packet list.
+
+  Raises:
+    ValueError: If the Packet doesn't contain std:vector<Packet>.
+
+  Examples:
+    packet = mp.packet_creator.create_packet_vector([
+        packet_creator.create_float(0.1),
+        packet_creator.create_int(1),
+        packet_creator.create_string('1')
+    ])
+    packet_list = mp.packet_getter.get_packet_list(packet)
+)doc");
+
+  m->def(
+      "get_str_to_packet_dict", &GetContent<std::map<std::string, Packet>>,
+
+      R"doc(Get the content of a MediaPipe Packet as a dictionary that has (str, Packet) pairs.
+
+  Args:
+    packet: A MediaPipe Packet that holds std::map<std::string, Packet>.
+
+  Returns:
+    A dictionary that has (str, Packet) pairs.
+
+  Raises:
+    ValueError: If the Packet doesn't contain std::map<std::string, Packet>.
+
+  Examples:
+    dict_packet = mp.packet_creator.create_string_to_packet_map({
+        'float': packet_creator.create_float(0.1),
+        'int': packet_creator.create_int(1),
+        'std::string': packet_creator.create_string('1')
+    data = mp.packet_getter.get_str_to_packet_dict(dict_packet)
+)doc");
+
+  m->def(
+      "get_image_frame", &GetContent<ImageFrame>,
+      R"doc(Get the content of a MediaPipe ImageFrame Packet as an ImageFrame object.
+
+  Args:
+    packet: A MediaPipe ImageFrame Packet.
+
+  Returns:
+    A MediaPipe ImageFrame object.
+
+  Raises:
+    ValueError: If the Packet doesn't contain ImageFrame.
+
+  Examples:
+    packet = packet_creator.create_image_frame(frame)
+    data = packet_getter.get_image_frame(packet)
+)doc",
+      py::return_value_policy::reference_internal);
+
+  m->def(
+      "get_matrix",
+      [](const Packet& packet) {
+        return Eigen::Ref<const Eigen::MatrixXf>(GetContent<Matrix>(packet));
+      },
+      R"doc(Get the content of a MediaPipe Matrix Packet as a numpy 2d float ndarray.
+
+  Args:
+    packet: A MediaPipe Matrix Packet.
+
+  Returns:
+    A numpy 2d float ndarray.
+
+  Raises:
+    ValueError: If the Packet doesn't contain matrix data.
+
+  Examples:
+    packet = mp.packet_creator.create_matrix(2d_array)
+    data = mp.packet_getter.get_matrix(packet)
+)doc",
+      py::return_value_policy::reference_internal);
+}
+
+void InternalPacketGetters(pybind11::module* m) {
+  m->def(
+      "_get_proto_type_name",
+      [](const Packet& packet) {
+        return packet.GetProtoMessageLite().GetTypeName();
+      },
+      py::return_value_policy::move);
+
+  m->def(
+      "_get_proto_vector_size",
+      [](Packet& packet) {
+        auto proto_vector = packet.GetVectorOfProtoMessageLitePtrs();
+        RaisePyErrorIfNotOk(proto_vector.status());
+        return proto_vector.ValueOrDie().size();
+      },
+      py::return_value_policy::move);
+
+  m->def(
+      "_get_proto_vector_element_type_name",
+      [](Packet& packet) {
+        auto proto_vector = packet.GetVectorOfProtoMessageLitePtrs();
+        RaisePyErrorIfNotOk(proto_vector.status());
+        if (proto_vector.ValueOrDie().empty()) {
+          return std::string();
+        }
+        return proto_vector.ValueOrDie()[0]->GetTypeName();
+      },
+      py::return_value_policy::move);
+
+  m->def(
+      "_get_serialized_proto",
+      [](const Packet& packet) {
+        // By default, py::bytes is an extra copy of the original std::string
+        // object: https://github.com/pybind/pybind11/issues/1236 Howeover, when
+        // Pybind11 performs the C++ to Python transition, it only increases the
+        // py::bytes object's ref count. See the implmentation at line 1583 in
+        // "pybind11/cast.h".
+        return py::bytes(packet.GetProtoMessageLite().SerializeAsString());
+      },
+      py::return_value_policy::move);
+
+  m->def(
+      "_get_serialized_proto_list",
+      [](Packet& packet) {
+        auto proto_vector = packet.GetVectorOfProtoMessageLitePtrs();
+        RaisePyErrorIfNotOk(proto_vector.status());
+        int size = proto_vector.ValueOrDie().size();
+        std::vector<py::bytes> results;
+        results.reserve(size);
+        for (const proto_ns::MessageLite* ptr : proto_vector.ValueOrDie()) {
+          results.push_back(py::bytes(ptr->SerializeAsString()));
+        }
+        return results;
+      },
+      py::return_value_policy::move);
+}
+
+void PacketGetterSubmodule(pybind11::module* module) {
+  py::module m = module->def_submodule(
+      "_packet_getter", "MediaPipe internal packet getter module.");
+  PublicPacketGetters(&m);
+  InternalPacketGetters(&m);
+}
+
+}  // namespace python
+}  // namespace mediapipe

mediapipe/python/pybind/packet_getter.h

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_PACKET_GETTER_H_
+#define MEDIAPIPE_PYTHON_PYBIND_PACKET_GETTER_H_
+
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+void PacketGetterSubmodule(pybind11::module* module);
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_PACKET_GETTER_H_

mediapipe/python/pybind/resource_util.cc

@@ -0,0 +1,47 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/python/pybind/resource_util.h"
+
+#include "absl/flags/declare.h"
+#include "absl/flags/flag.h"
+
+ABSL_DECLARE_FLAG(std::string, resource_root_dir);
+
+namespace mediapipe {
+namespace python {
+
+namespace py = pybind11;
+
+void ResourceUtilSubmodule(pybind11::module* module) {
+  py::module m =
+      module->def_submodule("resource_util", "MediaPipe resource util module.");
+
+  m.def(
+      "set_resource_dir",
+      [](const std::string& str) {
+        absl::SetFlag(&FLAGS_resource_root_dir, str);
+      },
+      R"doc(Set resource root directory where can find necessary graph resources such as model files and label maps.
+
+  Args:
+    str: A UTF-8 str.
+
+  Examples:
+    mp.resource_util.set_resource_dir('/path/to/resource')
+)doc");
+}
+
+}  // namespace python
+}  // namespace mediapipe

mediapipe/python/pybind/resource_util.h

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_RESOURCE_UTIL_H_
+#define MEDIAPIPE_PYTHON_PYBIND_RESOURCE_UTIL_H_
+
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+void ResourceUtilSubmodule(pybind11::module* module);
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_RESOURCE_UTIL_H_

mediapipe/python/pybind/timestamp.cc

@@ -0,0 +1,144 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/python/pybind/timestamp.h"
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/python/pybind/util.h"
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+namespace py = pybind11;
+
+void TimestampSubmodule(pybind11::module* module) {
+  py::module m =
+      module->def_submodule("timestamp", "MediaPipe timestamp module.");
+
+  py::class_<Timestamp> timestamp(
+      m, "Timestamp",
+      R"doc(A class which represents a timestamp in the MediaPipe framework.
+
+  MediaPipe timestamps are in units of _microseconds_.
+  There are several special values (All these values must be constructed using
+  the static methods provided):
+    UNSET:       The default initialization value, not generally valid when a
+                 timestamp is required.
+    UNSTARTED:   The timestamp before any valid timestamps. This is the input
+                 timestamp during Calcultor::Open().
+    PRESTREAM:   A value for specifying that a packet contains "header" data
+                 that should be processed before any other timestamp.  Like
+                 poststream, if this value is sent then it must be the only
+                 value that is sent on the stream.
+    MIN:         The minimum range timestamp to see in Calcultor::Process().
+                 Any number of "range" timestamp can be sent over a stream,
+                 provided that they are sent in monotonically increasing order.
+    MAX:         The maximum range timestamp to see in Process().
+    POSTSTREAM:  A value for specifying that a packet pertains to the entire
+                 stream.  This "summary" timestamp occurs after all the "range"
+                 timestamps.  If this timestamp is sent on a stream, it must be
+                 the only packet sent.
+    DONE:        The timestamp after all valid timestamps.
+                 This is the input timestamp during Calcultor::Close().
+)doc");
+
+  timestamp.def(py::init<const Timestamp&>())
+      .def(py::init<int64>())
+      .def_property_readonly("value", &Timestamp::Value)
+      .def_property_readonly_static(
+          "UNSET", [](py::object) { return Timestamp::Unset(); })
+      .def_property_readonly_static(
+          "UNSTARTED", [](py::object) { return Timestamp::Unstarted(); })
+      .def_property_readonly_static(
+          "PRESTREAM", [](py::object) { return Timestamp::PreStream(); })
+      .def_property_readonly_static("MIN",
+                                    [](py::object) { return Timestamp::Min(); })
+      .def_property_readonly_static("MAX",
+                                    [](py::object) { return Timestamp::Max(); })
+      .def_property_readonly_static(
+          "POSTSTREAM", [](py::object) { return Timestamp::PostStream(); })
+      .def_property_readonly_static(
+          "DONE", [](py::object) { return Timestamp::Done(); })
+      .def("__eq__",
+           [](const Timestamp& a, const Timestamp& b) { return a == b; })
+      .def("__lt__",
+           [](const Timestamp& a, const Timestamp& b) { return a < b; })
+      .def("__gt__",
+           [](const Timestamp& a, const Timestamp& b) { return a > b; })
+      .def("__le__",
+           [](const Timestamp& a, const Timestamp& b) { return a <= b; })
+      .def("__ge__",
+           [](const Timestamp& a, const Timestamp& b) { return a >= b; })
+      .def("__repr__", [](const Timestamp& self) {
+        return absl::StrCat("<mediapipe.Timestamp with value: ",
+                            TimestampValueString(self), ">");
+      });
+
+  timestamp.def("seconds", &Timestamp::Seconds,
+                R"doc(Return the value in units of seconds as a float.)doc");
+
+  timestamp.def(
+      "microseconds", &Timestamp::Microseconds,
+      R"doc(Return the value in units of microseconds as an int.)doc");
+
+  timestamp.def("is_special_value", &Timestamp::IsSpecialValue,
+                R"doc(Check if the timestamp is a special value,
+
+    A special value is any of the values which cannot be constructed directly
+    but must be constructed using the static special value.
+
+)doc");
+
+  timestamp.def(
+      "is_range_value", &Timestamp::IsRangeValue,
+      R"doc(Check if the timestamp is a range value is anything between Min() and Max() (inclusive).
+
+  Any number of packets with range values can be sent over a stream as long as
+  they are sent in monotonically increasing order. is_range_value() isn't
+  quite the opposite of is_special_value() since it is valid to start a stream
+  at Timestamp::Min() and continue until timestamp max (both of which are
+  special values). prestream and postStream  are not considered a range value
+  even though they can be sent over a stream (they are "summary" timestamps not
+  "range" timestamps).
+)doc");
+
+  timestamp.def(
+      "is_allowed_in_stream", &Timestamp::IsAllowedInStream,
+      R"doc(Returns true iff this can be the timestamp of a Packet in a stream.
+
+  Any number of RangeValue timestamps may be in a stream (in monotonically
+  increasing order).  Also, exactly one prestream, or one poststream packet is
+  allowed.
+)doc");
+
+  timestamp.def_static("from_seconds", &Timestamp::FromSeconds,
+                       R"doc(Create a timestamp from a seconds value
+
+  Args:
+    seconds: A seconds value in float.
+
+  Returns:
+    A MediaPipe Timestamp object.
+
+  Examples:
+    timestamp_now = mp.Timestamp.from_seconds(time.time())
+)doc");
+
+  py::implicitly_convertible<int64, Timestamp>();
+}
+
+}  // namespace python
+}  // namespace mediapipe

mediapipe/python/pybind/timestamp.h

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_TIMESTAMP_H_
+#define MEDIAPIPE_PYTHON_PYBIND_TIMESTAMP_H_
+
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+void TimestampSubmodule(pybind11::module* module);
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_TIMESTAMP_H_

mediapipe/python/pybind/util.h

@@ -0,0 +1,92 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PYTHON_PYBIND_UTIL_H_
+#define MEDIAPIPE_PYTHON_PYBIND_UTIL_H_
+
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
+#include "pybind11/pybind11.h"
+
+namespace mediapipe {
+namespace python {
+
+namespace py = pybind11;
+
+inline py::error_already_set RaisePyError(PyObject* exc_class,
+                                          const char* message) {
+  PyErr_SetString(exc_class, message);
+  return py::error_already_set();
+}
+
+inline PyObject* StatusCodeToPyError(const ::absl::StatusCode& code) {
+  switch (code) {
+    case absl::StatusCode::kInvalidArgument:
+      return static_cast<PyObject*>(PyExc_ValueError);
+    case absl::StatusCode::kAlreadyExists:
+      return static_cast<PyObject*>(PyExc_FileExistsError);
+    case absl::StatusCode::kUnimplemented:
+      return static_cast<PyObject*>(PyExc_NotImplementedError);
+    default:
+      return static_cast<PyObject*>(PyExc_RuntimeError);
+  }
+}
+
+inline void RaisePyErrorIfNotOk(const mediapipe::Status& status) {
+  if (!status.ok()) {
+    throw RaisePyError(StatusCodeToPyError(status.code()),
+                       status.message().data());
+  }
+}
+
+inline void RaisePyErrorIfOverflow(int64 value, int64 min, int64 max) {
+  if (value > max) {
+    throw RaisePyError(PyExc_OverflowError,
+                       absl::StrCat(value, " execeeds the maximum value (", max,
+                                    ") the data type can have.")
+                           .c_str());
+  } else if (value < min) {
+    throw RaisePyError(PyExc_OverflowError,
+                       absl::StrCat(value, " goes below the minimum value (",
+                                    min, ") the data type can have.")
+                           .c_str());
+  }
+}
+
+inline std::string TimestampValueString(const Timestamp& timestamp) {
+  if (timestamp == Timestamp::Unset()) {
+    return "UNSET";
+  } else if (timestamp == Timestamp::Unstarted()) {
+    return "UNSTARTED";
+  } else if (timestamp == Timestamp::PreStream()) {
+    return "PRESTREAM";
+  } else if (timestamp == Timestamp::Min()) {
+    return "MIN";
+  } else if (timestamp == Timestamp::Max()) {
+    return "MAX";
+  } else if (timestamp == Timestamp::PostStream()) {
+    return "POSTSTREAM";
+  } else if (timestamp == Timestamp::OneOverPostStream()) {
+    return "ONEOVERPOSTSTREAM";
+  } else if (timestamp == Timestamp::Done()) {
+    return "DONE";
+  } else {
+    return timestamp.DebugString();
+  }
+}
+
+}  // namespace python
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_PYTHON_PYBIND_UTIL_H_

mediapipe/python/timestamp_test.py

@@ -0,0 +1,75 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for mediapipe.python._framework_bindings.timestamp."""
+
+import time
+
+from absl.testing import absltest
+import mediapipe.python as mp
+
+
+class TimestampTest(absltest.TestCase):
+
+  def testTimesatmp(self):
+    t = mp.Timestamp(100)
+    self.assertEqual(t.value, 100)
+    self.assertEqual(t, 100)
+    self.assertEqual(str(t), '<mediapipe.Timestamp with value: 100>')
+
+  def testTimestampCopyConstructor(self):
+    ts1 = mp.Timestamp(100)
+    ts2 = mp.Timestamp(ts1)
+    self.assertEqual(ts1, ts2)
+
+  def testTimesatmpComparsion(self):
+    ts1 = mp.Timestamp(100)
+    ts2 = mp.Timestamp(100)
+    self.assertEqual(ts1, ts2)
+    ts3 = mp.Timestamp(200)
+    self.assertNotEqual(ts1, ts3)
+
+  def testTimesatmpSpecialValues(self):
+    t1 = mp.Timestamp.UNSET
+    self.assertEqual(str(t1), '<mediapipe.Timestamp with value: UNSET>')
+    t2 = mp.Timestamp.UNSTARTED
+    self.assertEqual(str(t2), '<mediapipe.Timestamp with value: UNSTARTED>')
+    t3 = mp.Timestamp.PRESTREAM
+    self.assertEqual(str(t3), '<mediapipe.Timestamp with value: PRESTREAM>')
+    t4 = mp.Timestamp.MIN
+    self.assertEqual(str(t4), '<mediapipe.Timestamp with value: MIN>')
+    t5 = mp.Timestamp.MAX
+    self.assertEqual(str(t5), '<mediapipe.Timestamp with value: MAX>')
+    t6 = mp.Timestamp.POSTSTREAM
+    self.assertEqual(str(t6), '<mediapipe.Timestamp with value: POSTSTREAM>')
+    t7 = mp.Timestamp.DONE
+    self.assertEqual(str(t7), '<mediapipe.Timestamp with value: DONE>')
+
+  def testTimestampComparisons(self):
+    ts1 = mp.Timestamp(100)
+    ts2 = mp.Timestamp(101)
+    self.assertGreater(ts2, ts1)
+    self.assertGreaterEqual(ts2, ts1)
+    self.assertLess(ts1, ts2)
+    self.assertLessEqual(ts1, ts2)
+    self.assertNotEqual(ts1, ts2)
+
+  def testFromSeconds(self):
+    now = time.time()
+    ts = mp.Timestamp.from_seconds(now)
+    self.assertAlmostEqual(now, ts.seconds(), delta=1)
+
+
+if __name__ == '__main__':
+  absltest.main()