Project import generated by Copybara.

PiperOrigin-RevId: 253489161

mediapipe/examples/desktop/BUILD

@@ -0,0 +1,31 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_library(
+    name = "simple_run_graph_main",
+    srcs = ["simple_run_graph_main.cc"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:map_util",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)

mediapipe/examples/desktop/README.md

@@ -0,0 +1,48 @@
+**Hello World**
+
+To build the "Hello World" example, use:
+
+```
+bazel build -c opt mediapipe/examples/desktop/hello_world:hello_world
+```
+
+and then run it using:
+
+```
+bazel-bin/mediapipe/examples/desktop/hello_world/hello_world --logtostderr
+```
+
+**TFlite Object Detection**
+
+To build the objet detection demo using a TFLite model on desktop, use:
+
+```
+bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_tflite --define 'MEDIAPIPE_DISABLE_GPU=1'
+```
+
+and run it using:
+
+```
+bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
+  --calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt \
+  --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file \
+  --alsologtostderr
+```
+
+**TensorFlow Object Detection**
+
+To build the object detection demo using a TensorFlow model on desktop, use:
+
+```
+bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_tensorflow \
+  --define 'MEDIAPIPE_DISABLE_GPU=1'
+```
+
+and run it using:
+
+```
+bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tensorflow  \
+  --calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt  \
+  --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
+  --alsologtostderr
+```

mediapipe/examples/desktop/__init__.py

@@ -0,0 +1,14 @@
+"""Copyright 2019 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

mediapipe/examples/desktop/hello_world/BUILD

@@ -0,0 +1,30 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "hello_world",
+    srcs = ["hello_world.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/calculators/core:pass_through_calculator",
+        "//mediapipe/framework:calculator_graph",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)

mediapipe/examples/desktop/hello_world/hello_world.cc

@@ -0,0 +1,65 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// A simple example to print out "Hello World!" from a MediaPipe graph.
+
+#include "mediapipe/framework/calculator_graph.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+::mediapipe::Status PrintHelloWorld() {
+  // Configures a simple graph, which concatenates 2 PassThroughCalculators.
+  CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "in"
+    output_stream: "out"
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "in"
+      output_stream: "out1"
+    }
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "out1"
+      output_stream: "out"
+    }
+  )");
+
+  CalculatorGraph graph;
+  RETURN_IF_ERROR(graph.Initialize(config));
+  ASSIGN_OR_RETURN(OutputStreamPoller poller,
+                   graph.AddOutputStreamPoller("out"));
+  RETURN_IF_ERROR(graph.StartRun({}));
+  // Give 10 input packets that contains the same std::string "Hello World!".
+  for (int i = 0; i < 10; ++i) {
+    RETURN_IF_ERROR(graph.AddPacketToInputStream(
+        "in", MakePacket<std::string>("Hello World!").At(Timestamp(i))));
+  }
+  // Close the input stream "in".
+  RETURN_IF_ERROR(graph.CloseInputStream("in"));
+  mediapipe::Packet packet;
+  // Get the output packets std::string.
+  while (poller.Next(&packet)) {
+    LOG(INFO) << packet.Get<std::string>();
+  }
+  return graph.WaitUntilDone();
+}
+}  // namespace mediapipe
+
+int main(int argc, char** argv) {
+  CHECK(mediapipe::PrintHelloWorld().ok());
+  return 0;
+}

mediapipe/examples/desktop/media_sequence/BUILD

@@ -0,0 +1,39 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_library(
+    name = "run_graph_file_io_main",
+    srcs = ["run_graph_file_io_main.cc"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:map_util",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "media_sequence_demo",
+    deps = [
+        ":run_graph_file_io_main",
+        "//mediapipe/graphs/media_sequence:clipped_images_from_file_at_24fps_calculators",
+    ],
+)

mediapipe/examples/desktop/media_sequence/README.md

@@ -0,0 +1,51 @@
+# Preparing data sets for machine learning with MediaPipe
+We include two pipelines to prepare data sets for training TensorFlow models.
+
+Using these data sets is split into two parts. First, the data set is
+constructed in with a Python script and MediaPipe C++ binary. The C++ binary
+should be compiled by the end user because the preparation for different data
+sets requires different MediaPipe calculator dependencies. The result of running
+the script is a data set of TFRecord files on disk. The second stage is reading
+the data from TensorFlow into a tf.data.Dataset. Both pipelines can be imported
+and support a simple call to as_dataset() to make the data available.
+
+### Demo data set
+To generate the demo dataset you must have Tensorflow [version >= 1.19]
+installed. Then the media_sequence_demo binary must be built from the top
+directory in the mediapipe repo and the command to build the data set must be
+run from the same directory.
+```
+bazel -c opt mediapipe/examples/desktop/media_sequence:media_sequence_demo \
+  --define=MEDIAPIPE_DISABLE_GPU=1
+
+python -m mediapipe.examples.desktop.media_sequence.demo_dataset \
+  --alsologtostderr \
+  --path_to_demo_data=/tmp/demo_data/ \
+  --path_to_mediapipe_binary=bazel-bin/mediapipe/examples/desktop/\
+media_sequence/media_sequence_demo  \
+  --path_to_graph_directory=mediapipe/graphs/media_sequence/
+```
+
+### Charades data set
+
+The Charades data set is ready for training and/or evaluating action recognition
+models in TensorFlow. You may only use this script in ways that comply with the
+Allen Institute for Artificial Intelligence's [license for the Charades data
+set.](https://allenai.org/plato/charades/license.txt)
+
+To generate the Charades dataset you must have Tensorflow [version >= 1.19]
+installed. Then the media_sequence_demo binary must be built from the top
+directory in the mediapipe repo and the command to build the data set must be
+run from the same directory.
+
+```
+bazel -c opt mediapipe/examples/desktop/media_sequence:media_sequence_demo \
+  --define=MEDIAPIPE_DISABLE_GPU=1
+
+python -m mediapipe.examples.desktop.media_sequence.charades_dataset \
+  --alsologtostderr \
+  --path_to_charades_data=/tmp/charades_data/ \
+  --path_to_mediapipe_binary=bazel-bin/mediapipe/examples/desktop/\
+media_sequence/media_sequence_demo  \
+  --path_to_graph_directory=mediapipe/graphs/media_sequence/
+```

mediapipe/examples/desktop/media_sequence/__init__.py

@@ -0,0 +1,14 @@
+"""Copyright 2019 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

mediapipe/examples/desktop/media_sequence/charades_dataset.py

@@ -0,0 +1,517 @@
+r"""Copyright 2019 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Code to download and parse the Charades dataset for TensorFlow models.
+
+The [Charades data set](https://allenai.org/plato/charades/) is a data set of
+human action recognition collected with and maintained by the Allen Institute
+for Artificial Intelligence. This script downloads and prepares the data set for
+training a TensorFlow model. To use this script, you must abide by the
+[lincense](https://allenai.org/plato/charades/license.txt) for the Charades data
+set provided by the Allen Institute. The license for this script only covers
+this code and not the data set.
+
+Running this code as a module generates the data set on disk. First, the
+required files are downloaded (_download_data). Then, for each split in the
+data set (generate_examples), the metadata is generated from the annotations for
+each example (_generate_metadata), and MediaPipe is used to fill in the video
+frames (_run_mediapipe). The data set is written to disk as a set of numbered
+TFRecord files. If the download is disrupted, the incomplete files will need to
+be removed before running the script again. This pattern can be reproduced and
+modified to generate most video data sets.
+
+Generating the data on disk will probably take 4-8 hours and requires 150 GB of
+disk space. (Image compression quality is the primary determiner of disk usage.)
+After generating the data, the 30 GB of compressed video data can be deleted.
+
+Once the data is on disk, reading the data as a tf.data.Dataset is accomplished
+with the following lines:
+
+   charades = CharadesDataset("charades_data_path")
+   dataset = charades.as_dataset("test")
+   # implement additional processing and batching here
+   images_and_labels = dataset.make_one_shot_iterator().get_next()
+   images = images_and_labels["images"]
+   labels = image_and_labels["classification_target"]
+   label_weights = image_and_labels["indicator_matrix"]
+
+This data is structured for per-frame action classification where images is
+the sequence of images, labels are the sequence of classification targets and,
+label_weights is 1 for valid frames and 0 for padded frames (if any). See
+as_dataset() for more details.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import contextlib
+import csv
+import os
+import random
+import subprocess
+import sys
+import tempfile
+import urllib
+import zipfile
+from absl import app
+from absl import flags
+from absl import logging
+import tensorflow as tf
+from mediapipe.util.sequence import media_sequence as ms
+
+
+DATA_URL_ANNOTATIONS = "http://ai2-website.s3.amazonaws.com/data/Charades.zip"
+DATA_URL_VIDEOS = "http://ai2-website.s3.amazonaws.com/data/Charades_v1_480.zip"
+DATA_URL_LICENSE = "https://allenai.org/plato/charades/license.txt"
+CITATION = r"""@article{sigurdsson2016hollywood,
+author = {Gunnar A. Sigurdsson and G{\"u}l Varol and Xiaolong Wang and Ivan Laptev and Ali Farhadi and Abhinav Gupta},
+title = {Hollywood in Homes: Crowdsourcing Data Collection for Activity Understanding},
+journal = {ArXiv e-prints},
+eprint = {1604.01753},
+year = {2016},
+url = {http://arxiv.org/abs/1604.01753},
+}"""
+SECONDS_TO_MICROSECONDS = 1000000
+GRAPHS = ["clipped_images_from_file_at_24fps.pbtxt"]
+SPLITS = {
+    "train": ("charades_v1_train_records",  # base name for sharded files
+              "Charades_v1_train.csv",  # path to csv of annotations
+              1000,  # number of shards
+              7986),  # number of examples
+    "test": ("charades_v1_test_records",
+             "Charades_v1_test.csv",
+             100,
+             1864),
+}
+NUM_CLASSES = 157
+CLASS_LABEL_OFFSET = 1
+
+
+class Charades(object):
+  """Generates and loads the Charades data set."""
+
+  def __init__(self, path_to_data):
+    if not path_to_data:
+      raise ValueError("You must supply the path to the data directory.")
+    self.path_to_data = path_to_data
+
+  def as_dataset(self, split, shuffle=False, repeat=False,
+                 serialized_prefetch_size=32, decoded_prefetch_size=32):
+    """Returns Charades as a tf.data.Dataset.
+
+    After running this function, calling padded_batch() on the Dataset object
+    will produce batches of data, but additional preprocessing may be desired.
+    If using padded_batch, the indicator_matrix output distinguishes valid
+    from padded frames.
+
+    Args:
+      split: either "train" or "test"
+      shuffle: if true, shuffles both files and examples.
+      repeat: if true, repeats the data set forever.
+      serialized_prefetch_size: the buffer size for reading from disk.
+      decoded_prefetch_size: the buffer size after decoding.
+    Returns:
+      A tf.data.Dataset object with the following structure: {
+        "images": uint8 tensor, shape [time, height, width, channels]
+        "segment_matrix": binary tensor of segments, shape [time, num_segments].
+          See one_hot_segments() for details.
+        "indicator_matrix": binary tensor indicating valid frames,
+          shape [time, 1]. If padded with zeros to align sizes, the indicator
+          marks where segments is valid.
+        "classification_target": binary tensor of classification targets,
+          shape [time, 158 classes]. More than one value in a row can be 1.0 if
+          segments overlap.
+        "example_id": a unique string id for each example, shape [].
+        "sampling_rate": the frame rate for each sequence, shape [].
+        "gt_segment_seconds": the start and end time of each segment,
+          shape [num_segments, 2].
+        "gt_segment_classes": the class labels for each segment,
+          shape [num_segments].
+        "num_segments": the number of segments in the example, shape [].
+        "num_timesteps": the number of timesteps in the example, shape [].
+        "images": the [time, height, width, channels] tensor of images.
+    """
+    def parse_fn(sequence_example):
+      """Parses a Charades example."""
+      context_features = {
+          ms.get_example_id_key(): ms.get_example_id_default_parser(),
+          ms.get_segment_start_index_key(): (
+              ms.get_segment_start_index_default_parser()),
+          ms.get_segment_end_index_key(): (
+              ms.get_segment_end_index_default_parser()),
+          ms.get_segment_label_index_key(): (
+              ms.get_segment_label_index_default_parser()),
+          ms.get_segment_label_string_key(): (
+              ms.get_segment_label_string_default_parser()),
+          ms.get_segment_start_timestamp_key(): (
+              ms.get_segment_start_timestamp_default_parser()),
+          ms.get_segment_end_timestamp_key(): (
+              ms.get_segment_end_timestamp_default_parser()),
+          ms.get_image_frame_rate_key(): (
+              ms.get_image_frame_rate_default_parser()),
+      }
+
+      sequence_features = {
+          ms.get_image_encoded_key(): ms.get_image_encoded_default_parser()
+      }
+      parsed_context, parsed_sequence = tf.io.parse_single_sequence_example(
+          sequence_example, context_features, sequence_features)
+
+      sequence_length = tf.shape(parsed_sequence[ms.get_image_encoded_key()])[0]
+      num_segments = tf.shape(
+          parsed_context[ms.get_segment_label_index_key()])[0]
+      # segments matrix and targets for training.
+      segments_matrix, indicator = one_hot_segments(
+          tf.sparse_tensor_to_dense(
+              parsed_context[ms.get_segment_start_index_key()]),
+          tf.sparse_tensor_to_dense(
+              parsed_context[ms.get_segment_end_index_key()]),
+          sequence_length)
+
+      classification_target = timepoint_classification_target(
+          segments_matrix,
+          tf.sparse_tensor_to_dense(
+              parsed_context[ms.get_segment_label_index_key()]
+              ) + CLASS_LABEL_OFFSET,
+          NUM_CLASSES + CLASS_LABEL_OFFSET)
+
+      # [segments, 2] start and end time in seconds.
+      gt_segment_seconds = tf.to_float(tf.concat(
+          [tf.expand_dims(tf.sparse_tensor_to_dense(parsed_context[
+              ms.get_segment_start_timestamp_key()]), 1),
+           tf.expand_dims(tf.sparse_tensor_to_dense(parsed_context[
+               ms.get_segment_end_timestamp_key()]), 1)],
+          1)) / float(SECONDS_TO_MICROSECONDS)
+      gt_segment_classes = tf.sparse_tensor_to_dense(parsed_context[
+          ms.get_segment_label_index_key()]) + CLASS_LABEL_OFFSET
+      example_id = parsed_context[ms.get_example_id_key()]
+      sampling_rate = parsed_context[ms.get_image_frame_rate_key()]
+
+      images = tf.map_fn(tf.image.decode_jpeg,
+                         parsed_sequence[ms.get_image_encoded_key()],
+                         back_prop=False,
+                         dtype=tf.uint8)
+
+      output_dict = {
+          "segment_matrix": segments_matrix,
+          "indicator_matrix": indicator,
+          "classification_target": classification_target,
+          "example_id": example_id,
+          "sampling_rate": sampling_rate,
+          "gt_segment_seconds": gt_segment_seconds,
+          "gt_segment_classes": gt_segment_classes,
+          "num_segments": num_segments,
+          "num_timesteps": sequence_length,
+          "images": images,
+      }
+      return output_dict
+
+    if split not in SPLITS:
+      raise ValueError("Split %s not in %s" % split, str(SPLITS.keys()))
+    all_shards = tf.io.gfile.glob(
+        os.path.join(self.path_to_data, SPLITS[split][0] + "-*-of-*"))
+    random.shuffle(all_shards)
+    all_shards_dataset = tf.data.Dataset.from_tensor_slices(all_shards)
+    cycle_length = min(16, len(all_shards))
+    dataset = all_shards_dataset.apply(
+        tf.contrib.data.parallel_interleave(
+            tf.data.TFRecordDataset,
+            cycle_length=cycle_length,
+            block_length=1, sloppy=True,
+            buffer_output_elements=serialized_prefetch_size))
+    dataset = dataset.prefetch(serialized_prefetch_size)
+    if shuffle:
+      dataset = dataset.shuffle(serialized_prefetch_size)
+    if repeat:
+      dataset = dataset.repeat()
+    dataset = dataset.map(parse_fn)
+    dataset = dataset.prefetch(decoded_prefetch_size)
+    return dataset
+
+  def generate_examples(self,
+                        path_to_mediapipe_binary, path_to_graph_directory):
+    """Downloads data and generates sharded TFRecords.
+
+    Downloads the data files, generates metadata, and processes the metadata
+    with MediaPipe to produce tf.SequenceExamples for training. The resulting
+    files can be read with as_dataset(). After running this function the
+    original data files can be deleted.
+
+    Args:
+      path_to_mediapipe_binary: Path to the compiled binary for the BUILD target
+        mediapipe/examples/desktop/demo:media_sequence_demo.
+      path_to_graph_directory: Path to the directory with MediaPipe graphs in
+        mediapipe/graphs/media_sequence/.
+    """
+    if not path_to_mediapipe_binary:
+      raise ValueError(
+          "You must supply the path to the MediaPipe binary for "
+          "mediapipe/examples/desktop/demo:media_sequence_demo.")
+    if not path_to_graph_directory:
+      raise ValueError(
+          "You must supply the path to the directory with MediaPipe graphs in "
+          "mediapipe/graphs/media_sequence/.")
+    logging.info("Downloading data.")
+    annotation_dir, video_dir = self._download_data()
+    for name, annotations, shards, _ in SPLITS.values():
+      annotation_file = os.path.join(
+          annotation_dir, annotations)
+      logging.info("Generating metadata for split: %s", name)
+      all_metadata = list(self._generate_metadata(annotation_file, video_dir))
+      random.seed(47)
+      random.shuffle(all_metadata)
+      shard_names = [os.path.join(self.path_to_data, name + "-%05d-of-%05d" % (
+          i, shards)) for i in range(shards)]
+      writers = [tf.io.TFRecordWriter(shard_name) for shard_name in shard_names]
+      with _close_on_exit(writers) as writers:
+        for i, seq_ex in enumerate(all_metadata):
+          print("Processing example %d of %d   (%d%%) \r" % (
+              i, len(all_metadata), i * 100 / len(all_metadata)), end="")
+          for graph in GRAPHS:
+            graph_path = os.path.join(path_to_graph_directory, graph)
+            seq_ex = self._run_mediapipe(
+                path_to_mediapipe_binary, seq_ex, graph_path)
+          writers[i % len(writers)].write(seq_ex.SerializeToString())
+    logging.info("Data extraction complete.")
+
+  def _generate_metadata(self, annotations_file, video_dir):
+    """For each row in the annotation CSV, generates the corresponding metadata.
+
+    Args:
+      annotations_file: path to the file of Charades CSV annotations.
+      video_dir: path to the directory of video files referenced by the
+        annotations.
+    Yields:
+      Each tf.SequenceExample of metadata, ready to pass to MediaPipe.
+    """
+    with open(annotations_file, "r") as annotations:
+      reader = csv.DictReader(annotations)
+      for row in reader:
+        metadata = tf.train.SequenceExample()
+        filepath = os.path.join(video_dir, "%s.mp4" % row["id"])
+        actions = row["actions"].split(";")
+        action_indices = []
+        action_strings = []
+        action_start_times = []
+        action_end_times = []
+        for action in actions:
+          if not action:
+            continue
+          string, start, end = action.split(" ")
+          action_indices.append(int(string[1:]))
+          action_strings.append(bytes23(string))
+          action_start_times.append(int(float(start) * SECONDS_TO_MICROSECONDS))
+          action_end_times.append(int(float(end) * SECONDS_TO_MICROSECONDS))
+        ms.set_example_id(bytes23(row["id"]), metadata)
+        ms.set_clip_data_path(bytes23(filepath), metadata)
+        ms.set_clip_start_timestamp(0, metadata)
+        ms.set_clip_end_timestamp(
+            int(float(row["length"]) * SECONDS_TO_MICROSECONDS), metadata)
+        ms.set_segment_start_timestamp(action_start_times, metadata)
+        ms.set_segment_end_timestamp(action_end_times, metadata)
+        ms.set_segment_label_string(action_strings, metadata)
+        ms.set_segment_label_index(action_indices, metadata)
+        yield metadata
+
+  def _download_data(self):
+    """Downloads and extracts data if not already available."""
+    if sys.version_info >= (3, 0):
+      urlretrieve = urllib.request.urlretrieve
+    else:
+      urlretrieve = urllib.urlretrieve
+    logging.info("Creating data directory.")
+    tf.io.gfile.makedirs(self.path_to_data)
+    logging.info("Downloading license.")
+    local_license_path = os.path.join(
+        self.path_to_data, DATA_URL_LICENSE.split("/")[-1])
+    if not tf.io.gfile.exists(local_license_path):
+      urlretrieve(DATA_URL_LICENSE, local_license_path)
+    logging.info("Downloading annotations.")
+    local_annotations_path = os.path.join(
+        self.path_to_data, DATA_URL_ANNOTATIONS.split("/")[-1])
+    if not tf.io.gfile.exists(local_annotations_path):
+      urlretrieve(DATA_URL_ANNOTATIONS, local_annotations_path)
+    logging.info("Downloading videos.")
+    local_videos_path = os.path.join(
+        self.path_to_data, DATA_URL_VIDEOS.split("/")[-1])
+    if not tf.io.gfile.exists(local_videos_path):
+      urlretrieve(DATA_URL_VIDEOS, local_videos_path, progress_hook)
+    logging.info("Extracting annotations.")
+    # return video dir and annotation_dir by removing .zip from the path.
+    annotations_dir = local_annotations_path[:-4]
+    if not tf.io.gfile.exists(annotations_dir):
+      with zipfile.ZipFile(local_annotations_path) as annotations_zip:
+        annotations_zip.extractall(self.path_to_data)
+    logging.info("Extracting videos.")
+    video_dir = local_videos_path[:-4]
+    if not tf.io.gfile.exists(video_dir):
+      with zipfile.ZipFile(local_videos_path) as videos_zip:
+        videos_zip.extractall(self.path_to_data)
+    return annotations_dir, video_dir
+
+  def _run_mediapipe(self, path_to_mediapipe_binary, sequence_example, graph):
+    """Runs MediaPipe over MediaSequence tf.train.SequenceExamples.
+
+    Args:
+      path_to_mediapipe_binary: Path to the compiled binary for the BUILD target
+        mediapipe/examples/desktop/demo:media_sequence_demo.
+      sequence_example: The SequenceExample with metadata or partial data file.
+      graph: The path to the graph that extracts data to add to the
+        SequenceExample.
+    Returns:
+      A copy of the input SequenceExample with additional data fields added
+      by the MediaPipe graph.
+    Raises:
+      RuntimeError: if MediaPipe returns an error or fails to run the graph.
+    """
+    if not path_to_mediapipe_binary:
+      raise ValueError("--path_to_mediapipe_binary must be specified.")
+    input_fd, input_filename = tempfile.mkstemp()
+    output_fd, output_filename = tempfile.mkstemp()
+    cmd = [path_to_mediapipe_binary,
+           "--calculator_graph_config_file=%s" % graph,
+           "--input_side_packets=input_sequence_example=%s" % input_filename,
+           "--output_side_packets=output_sequence_example=%s" % output_filename]
+    with open(input_filename, "wb") as input_file:
+      input_file.write(sequence_example.SerializeToString())
+    mediapipe_output = subprocess.check_output(cmd)
+    if b"Failed to run the graph" in mediapipe_output:
+      raise RuntimeError(mediapipe_output)
+    with open(output_filename, "rb") as output_file:
+      output_example = tf.train.SequenceExample()
+      output_example.ParseFromString(output_file.read())
+    os.close(input_fd)
+    os.remove(input_filename)
+    os.close(output_fd)
+    os.remove(output_filename)
+    return output_example
+
+
+def one_hot_segments(start_indices, end_indices, num_samples):
+  """Returns a one-hot, float matrix of segments at each timestep.
+
+  All integers in the inclusive range of start_indices and end_indices are used.
+  This allows start and end timestamps to be mapped to the same index and the
+  segment will not be omitted.
+
+  Args:
+    start_indices: a 1d tensor of integer indices for the start of each
+      segement.
+    end_indices: a tensor of integer indices for the end of each segment.
+      Must be the same shape as start_indices. Values should be >= start_indices
+      but not strictly enforced.
+    num_samples: the number of rows in the output. Indices should be <
+      num_samples, but this is not strictly enforced.
+  Returns:
+    (segments, indicator)
+    segments: A [num_samples, num_elements(start_indices)] tensor where in each
+      column the rows with indices >= start_indices[column] and
+      <= end_indices[column] are 1.0 and all other values are 0.0.
+    indicator: a tensor of 1.0 values with shape [num_samples, 1]. If padded
+      with zeros to align sizes, the indicator marks where segments is valid.
+  """
+  start_indices = tf.convert_to_tensor(start_indices)
+  end_indices = tf.convert_to_tensor(end_indices)
+  start_indices.shape.assert_is_compatible_with(end_indices.shape)
+  start_indices.shape.assert_has_rank(1)
+  end_indices.shape.assert_has_rank(1)
+  # create a matrix of the index at each row with a column per segment.
+  indices = tf.to_int64(
+      tf.tile(
+          tf.transpose(tf.expand_dims(tf.range(num_samples), 0)),
+          [1, tf.shape(start_indices)[0]]))
+  # switch to one hot encoding of segments (includes start and end indices)
+  segments = tf.to_float(
+      tf.logical_and(
+          tf.greater_equal(indices, start_indices),
+          tf.less_equal(indices, end_indices)))
+  # create a tensors of ones everywhere there's an annotation. If padded with
+  # zeros later, element-wise multiplication of the loss will mask out the
+  # padding.
+  indicator = tf.ones(shape=[num_samples, 1], dtype=tf.float32)
+  return segments, indicator
+
+
+def timepoint_classification_target(segments, segment_classes, num_classes):
+  """Produces a classification target at each timepoint.
+
+  If no segments are present at a time point, the first class is set to 1.0.
+  This should be used as a background class unless segments are always present.
+
+  Args:
+    segments: a [time, num_segments] tensor that is 1.0 at indices within
+      each segment and 0.0 elsewhere.
+    segment_classes: a [num_segments] tensor with the class index of each
+      segment.
+    num_classes: the number of classes (must be >= max(segment_classes) + 1)
+  Returns:
+    a [time, num_classes] tensor. In the final output, more than one
+    value in a row can be 1.0 if segments overlap.
+  """
+  num_segments = tf.shape(segments)[1]
+  matrix_of_class_indices = tf.to_int32(
+      segments * tf.to_float(tf.expand_dims(segment_classes, 0)))
+  # First column will have one count per zero segment. Correct this to be 0
+  # unless no segments are present.
+  one_hot = tf.reduce_sum(tf.one_hot(matrix_of_class_indices, num_classes), 1)
+  normalizer = tf.concat([
+      tf.ones(shape=[1, 1], dtype=tf.float32) / tf.to_float(num_segments),
+      tf.ones(shape=[1, num_classes - 1], dtype=tf.float32)
+  ], 1)
+  corrected_one_hot = tf.floor(one_hot * normalizer)
+  return corrected_one_hot
+
+
+def progress_hook(blocks, block_size, total_size):
+  print("Downloaded %d%% of %d bytes   (%d blocks)\r" % (
+      blocks * block_size / total_size * 100, total_size, blocks), end="")
+
+
+def bytes23(string):
+  """Creates a bytes string in either Python 2 or  3."""
+  if sys.version_info >= (3, 0):
+    return bytes(string, "utf8")
+  else:
+    return bytes(string)
+
+
+@contextlib.contextmanager
+def _close_on_exit(writers):
+  """Call close on all writers on exit."""
+  try:
+    yield writers
+  finally:
+    for writer in writers:
+      writer.close()
+
+
+def main(argv):
+  if len(argv) > 1:
+    raise app.UsageError("Too many command-line arguments.")
+  Charades(flags.FLAGS.path_to_charades_data).generate_examples(
+      flags.FLAGS.path_to_mediapipe_binary,
+      flags.FLAGS.path_to_graph_directory)
+
+if __name__ == "__main__":
+  flags.DEFINE_string("path_to_charades_data",
+                      "",
+                      "Path to directory to write data to.")
+  flags.DEFINE_string("path_to_mediapipe_binary",
+                      "",
+                      "Path to the MediaPipe run_graph_file_io_main binary.")
+  flags.DEFINE_string("path_to_graph_directory",
+                      "",
+                      "Path to directory containing the graph files.")
+  app.run(main)

mediapipe/examples/desktop/media_sequence/demo_dataset.py

@@ -0,0 +1,317 @@
+r"""Copyright 2019 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+A demo data set constructed with MediaSequence and MediaPipe.
+
+This code demonstrates the steps for constructing a data set with MediaSequence.
+This code has two functions. First, it can be run as a module to download and
+prepare a toy dataset. Second, it can be imported and used to provide a
+tf.data.Dataset reading that data from disk via as_dataset().
+
+Running as a module prepares the data in three stages via generate_examples().
+First, the actual data files are downloaded. If the download is disrupted, the
+incomplete files will need to be removed before running the script again.
+Second, the annotations are parsed and reformated into metadata as described in
+the MediaSequence documentation. Third, MediaPipe is run to extract subsequences
+of frames for subsequent training via _run_mediapipe().
+
+The toy data set is classifying a clip as a panning shot of galaxy or nebula
+from videos releasued under the [Creative Commons Attribution 4.0 International
+license](http://creativecommons.org/licenses/by/4.0/) on the ESA/Hubble site.
+(The use of these ESA/Hubble materials does not imply the endorsement by
+ESA/Hubble or any ESA/Hubble employee of a commercial product or service.) Each
+video is split into 5 or 6 ten-second clips with a label of "galaxy" or "nebula"
+and downsampled to 10 frames per second. (The last clip for each test example is
+only 6 seconds.) There is one video of each class in each of the training and
+testing splits.
+
+Reading the data as a tf.data.Dataset is accomplished with the following lines:
+
+   demo = DemoDataset("demo_data_path")
+   dataset = demo.as_dataset("test")
+   # implement additional processing and batching here
+   images_and_labels = dataset.make_one_shot_iterator().get_next()
+   images = images_and_labels["images"]
+   labels = image_and_labels["labels"]
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import contextlib
+import csv
+import os
+import random
+import subprocess
+import sys
+import tempfile
+import urllib
+
+from absl import app
+from absl import flags
+from absl import logging
+import tensorflow as tf
+
+from mediapipe.util.sequence import media_sequence as ms
+
+SPLITS = {
+    "train":
+        """url,label index,label string,duration,credits
+https://cdn.spacetelescope.org/archives/videos/medium_podcast/heic1608c.mp4,0,nebula,50,"ESA/Hubble; Music: Johan B. Monell"
+https://cdn.spacetelescope.org/archives/videos/medium_podcast/heic1712b.mp4,1,galaxy,50,"ESA/Hubble, Digitized Sky Survey, Nick Risinger (skysurvey.org) Music: Johan B Monell"
+""",
+    "test":
+        """url,label index,label string,duration,credits
+https://cdn.spacetelescope.org/archives/videos/medium_podcast/heic1301b.m4v,0,nebula,56,"NASA, ESA. Acknowledgement: Josh Lake"
+https://cdn.spacetelescope.org/archives/videos/medium_podcast/heic1305b.m4v,1,galaxy,56,"NASA, ESA, Digitized Sky Survey 2. Acknowledgement: A. van der Hoeven"
+"""
+}
+NUM_CLASSES = 2
+NUM_SHARDS = 2
+SECONDS_PER_EXAMPLE = 10
+MICROSECONDS_PER_SECOND = 1000000
+TF_RECORD_PATTERN = "demo_space_dataset_%s_tfrecord"
+GRAPHS = ["clipped_images_from_file_at_24fps.pbtxt"]
+
+
+class DemoDataset(object):
+  """Generates and loads a demo data set."""
+
+  def __init__(self, path_to_data):
+    if not path_to_data:
+      raise ValueError("You must supply the path to the data directory.")
+    self.path_to_data = path_to_data
+
+  def as_dataset(self,
+                 split,
+                 shuffle=False,
+                 repeat=False,
+                 serialized_prefetch_size=32,
+                 decoded_prefetch_size=32):
+    """Returns the dataset as a tf.data.Dataset.
+
+    Args:
+      split: either "train" or "test"
+      shuffle: if true, shuffles both files and examples.
+      repeat: if true, repeats the data set forever.
+      serialized_prefetch_size: the buffer size for reading from disk.
+      decoded_prefetch_size: the buffer size after decoding.
+
+    Returns:
+      A tf.data.Dataset object with the following structure: {
+        "images": uint8 tensor, shape [time, height, width, channels]
+        "labels": one hot encoded label tensor, shape [2]
+        "id": a unique string id for each example, shape []
+      }
+    """
+
+    def parse_fn(sequence_example):
+      """Parses a clip classification example."""
+      context_features = {
+          ms.get_example_id_key():
+              ms.get_example_id_default_parser(),
+          ms.get_clip_label_index_key():
+              ms.get_clip_label_index_default_parser(),
+          ms.get_clip_label_string_key():
+              ms.get_clip_label_string_default_parser()
+      }
+      sequence_features = {
+          ms.get_image_encoded_key(): ms.get_image_encoded_default_parser(),
+      }
+      parsed_context, parsed_sequence = tf.io.parse_single_sequence_example(
+          sequence_example, context_features, sequence_features)
+      example_id = parsed_context[ms.get_example_id_key()]
+      classification_target = tf.one_hot(
+          tf.sparse_tensor_to_dense(
+              parsed_context[ms.get_clip_label_index_key()]), NUM_CLASSES)
+      images = tf.map_fn(
+          tf.image.decode_jpeg,
+          parsed_sequence[ms.get_image_encoded_key()],
+          back_prop=False,
+          dtype=tf.uint8)
+      return {
+          "id": example_id,
+          "labels": classification_target,
+          "images": images,
+      }
+
+    if split not in SPLITS:
+      raise ValueError("split '%s' is unknown." % split)
+    all_shards = tf.io.gfile.glob(
+        os.path.join(self.path_to_data, TF_RECORD_PATTERN % split + "-*-of-*"))
+    if shuffle:
+      random.shuffle(all_shards)
+    all_shards_dataset = tf.data.Dataset.from_tensor_slices(all_shards)
+    cycle_length = min(16, len(all_shards))
+    dataset = all_shards_dataset.apply(
+        tf.contrib.data.parallel_interleave(
+            tf.data.TFRecordDataset,
+            cycle_length=cycle_length,
+            block_length=1,
+            sloppy=True,
+            buffer_output_elements=serialized_prefetch_size))
+    dataset = dataset.prefetch(serialized_prefetch_size)
+    if shuffle:
+      dataset = dataset.shuffle(serialized_prefetch_size)
+    if repeat:
+      dataset = dataset.repeat()
+    dataset = dataset.map(parse_fn)
+    dataset = dataset.prefetch(decoded_prefetch_size)
+    return dataset
+
+  def generate_examples(self, path_to_mediapipe_binary,
+                        path_to_graph_directory):
+    """Downloads data and generates sharded TFRecords.
+
+    Downloads the data files, generates metadata, and processes the metadata
+    with MediaPipe to produce tf.SequenceExamples for training. The resulting
+    files can be read with as_dataset(). After running this function the
+    original data files can be deleted.
+
+    Args:
+      path_to_mediapipe_binary: Path to the compiled binary for the BUILD target
+        mediapipe/examples/desktop/demo:media_sequence_demo.
+      path_to_graph_directory: Path to the directory with MediaPipe graphs in
+        mediapipe/graphs/media_sequence/.
+    """
+    if not path_to_mediapipe_binary:
+      raise ValueError("You must supply the path to the MediaPipe binary for "
+                       "mediapipe/examples/desktop/demo:media_sequence_demo.")
+    if not path_to_graph_directory:
+      raise ValueError(
+          "You must supply the path to the directory with MediaPipe graphs in "
+          "mediapipe/graphs/media_sequence/.")
+    logging.info("Downloading data.")
+    tf.io.gfile.makedirs(self.path_to_data)
+    if sys.version_info >= (3, 0):
+      urlretrieve = urllib.request.urlretrieve
+    else:
+      urlretrieve = urllib.urlretrieve
+    for split in SPLITS:
+      reader = csv.DictReader(SPLITS[split].split("\n"))
+      all_metadata = []
+      for row in reader:
+        url = row["url"]
+        basename = url.split("/")[-1]
+        local_path = os.path.join(self.path_to_data, basename)
+        if not tf.io.gfile.exists(local_path):
+          urlretrieve(url, local_path)
+
+        for start_time in range(0, int(row["duration"]), SECONDS_PER_EXAMPLE):
+          metadata = tf.train.SequenceExample()
+          ms.set_example_id(bytes23(basename + "_" + str(start_time)),
+                            metadata)
+          ms.set_clip_data_path(bytes23(local_path), metadata)
+          ms.set_clip_start_timestamp(start_time * MICROSECONDS_PER_SECOND,
+                                      metadata)
+          ms.set_clip_end_timestamp(
+              (start_time + SECONDS_PER_EXAMPLE) * MICROSECONDS_PER_SECOND,
+              metadata)
+          ms.set_clip_label_index((int(row["label index"]),), metadata)
+          ms.set_clip_label_string((bytes23(row["label string"]),),
+                                   metadata)
+          all_metadata.append(metadata)
+      random.seed(47)
+      random.shuffle(all_metadata)
+      shard_names = [self._indexed_shard(split, i) for i in range(NUM_SHARDS)]
+      writers = [tf.io.TFRecordWriter(shard_name) for shard_name in shard_names]
+      with _close_on_exit(writers) as writers:
+        for i, seq_ex in enumerate(all_metadata):
+          for graph in GRAPHS:
+            graph_path = os.path.join(path_to_graph_directory, graph)
+            seq_ex = self._run_mediapipe(path_to_mediapipe_binary, seq_ex,
+                                         graph_path)
+          writers[i % len(writers)].write(seq_ex.SerializeToString())
+
+  def _indexed_shard(self, split, index):
+    """Constructs a sharded filename."""
+    return os.path.join(
+        self.path_to_data,
+        TF_RECORD_PATTERN % split + "-%05d-of-%05d" % (index, NUM_SHARDS))
+
+  def _run_mediapipe(self, path_to_mediapipe_binary, sequence_example, graph):
+    """Runs MediaPipe over MediaSequence tf.train.SequenceExamples.
+
+    Args:
+      path_to_mediapipe_binary: Path to the compiled binary for the BUILD target
+        mediapipe/examples/desktop/demo:media_sequence_demo.
+      sequence_example: The SequenceExample with metadata or partial data file.
+      graph: The path to the graph that extracts data to add to the
+        SequenceExample.
+
+    Returns:
+      A copy of the input SequenceExample with additional data fields added
+      by the MediaPipe graph.
+    Raises:
+      RuntimeError: if MediaPipe returns an error or fails to run the graph.
+    """
+    if not path_to_mediapipe_binary:
+      raise ValueError("--path_to_mediapipe_binary must be specified.")
+    input_fd, input_filename = tempfile.mkstemp()
+    output_fd, output_filename = tempfile.mkstemp()
+    cmd = [
+        path_to_mediapipe_binary,
+        "--calculator_graph_config_file=%s" % graph,
+        "--input_side_packets=input_sequence_example=%s" % input_filename,
+        "--output_side_packets=output_sequence_example=%s" % output_filename
+    ]
+    with open(input_filename, "wb") as input_file:
+      input_file.write(sequence_example.SerializeToString())
+    mediapipe_output = subprocess.check_output(cmd)
+    if b"Failed to run the graph" in mediapipe_output:
+      raise RuntimeError(mediapipe_output)
+    with open(output_filename, "rb") as output_file:
+      output_example = tf.train.SequenceExample()
+      output_example.ParseFromString(output_file.read())
+    os.close(input_fd)
+    os.remove(input_filename)
+    os.close(output_fd)
+    os.remove(output_filename)
+    return output_example
+
+
+def bytes23(string):
+  """Creates a bytes string in either Python 2 or  3."""
+  if sys.version_info >= (3, 0):
+    return bytes(string, "utf8")
+  else:
+    return bytes(string)
+
+
+@contextlib.contextmanager
+def _close_on_exit(writers):
+  """Call close on all writers on exit."""
+  try:
+    yield writers
+  finally:
+    for writer in writers:
+      writer.close()
+
+
+def main(argv):
+  if len(argv) > 1:
+    raise app.UsageError("Too many command-line arguments.")
+  DemoDataset(flags.FLAGS.path_to_demo_data).generate_examples(
+      flags.FLAGS.path_to_mediapipe_binary, flags.FLAGS.path_to_graph_directory)
+
+
+if __name__ == "__main__":
+  flags.DEFINE_string("path_to_demo_data", "",
+                      "Path to directory to write data to.")
+  flags.DEFINE_string("path_to_mediapipe_binary", "",
+                      "Path to the MediaPipe run_graph_file_io_main binary.")
+  flags.DEFINE_string("path_to_graph_directory", "",
+                      "Path to directory containing the graph files.")
+  app.run(main)

mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc

@@ -0,0 +1,93 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// A simple main function to run a MediaPipe graph. Input side packets are read
+// from files provided via the command line and output side packets are written
+// to disk.
+
+#include "absl/strings/str_split.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/commandlineflags.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/map_util.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+
+DEFINE_string(
+    calculator_graph_config_file, "",
+    "Name of file containing text format CalculatorGraphConfig proto.");
+DEFINE_string(input_side_packets, "",
+              "Comma-separated list of key=value pairs specifying side packets "
+              "and corresponding file paths for the CalculatorGraph. The side "
+              "packets are read from the files and fed to the graph as strings "
+              "even if they represent doubles, floats, etc.");
+DEFINE_string(output_side_packets, "",
+              "Comma-separated list of key=value pairs specifying the output "
+              "side packets and paths to write to disk for the "
+              "CalculatorGraph.");
+
+::mediapipe::Status RunMediaPipeGraph() {
+  std::string calculator_graph_config_contents;
+  RETURN_IF_ERROR(mediapipe::file::GetContents(
+      FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
+  LOG(INFO) << "Get calculator graph config contents: "
+            << calculator_graph_config_contents;
+  mediapipe::CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
+          calculator_graph_config_contents);
+  std::map<std::string, ::mediapipe::Packet> input_side_packets;
+  std::vector<std::string> kv_pairs =
+      absl::StrSplit(FLAGS_input_side_packets, ',');
+  for (const std::string& kv_pair : kv_pairs) {
+    std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
+    RET_CHECK(name_and_value.size() == 2);
+    RET_CHECK(!::mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
+    std::string input_side_packet_contents;
+    RETURN_IF_ERROR(mediapipe::file::GetContents(name_and_value[1],
+                                                 &input_side_packet_contents));
+    input_side_packets[name_and_value[0]] =
+        ::mediapipe::MakePacket<std::string>(input_side_packet_contents);
+  }
+  LOG(INFO) << "Initialize the calculator graph.";
+  mediapipe::CalculatorGraph graph;
+  RETURN_IF_ERROR(graph.Initialize(config, input_side_packets));
+  LOG(INFO) << "Start running the calculator graph.";
+  RETURN_IF_ERROR(graph.Run());
+  LOG(INFO) << "Gathering output side packets.";
+  kv_pairs = absl::StrSplit(FLAGS_output_side_packets, ',');
+  for (const std::string& kv_pair : kv_pairs) {
+    std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
+    RET_CHECK(name_and_value.size() == 2);
+    ::mediapipe::StatusOr<::mediapipe::Packet> output_packet =
+        graph.GetOutputSidePacket(name_and_value[0]);
+    RET_CHECK(output_packet.ok())
+        << "Packet " << name_and_value[0] << " was not available.";
+    const std::string& serialized_string =
+        output_packet.ValueOrDie().Get<std::string>();
+    RETURN_IF_ERROR(
+        mediapipe::file::SetContents(name_and_value[1], serialized_string));
+  }
+  return ::mediapipe::OkStatus();
+}
+
+int main(int argc, char** argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  ::mediapipe::Status run_status = RunMediaPipeGraph();
+  if (!run_status.ok()) {
+    LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+  } else {
+    LOG(INFO) << "Success!";
+  }
+  return 0;
+}

mediapipe/examples/desktop/object_detection/BUILD

@@ -0,0 +1,70 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_library(
+    name = "object_detection_tensorflow_deps",
+    deps = [
+        "@org_tensorflow//tensorflow/c/kernels:bitcast_op",
+        "@org_tensorflow//tensorflow/core:direct_session",
+        "@org_tensorflow//tensorflow/core/kernels:argmax_op",
+        "@org_tensorflow//tensorflow/core/kernels:bias_op",
+        "@org_tensorflow//tensorflow/core/kernels:cast_op",
+        "@org_tensorflow//tensorflow/core/kernels:concat_op",
+        "@org_tensorflow//tensorflow/core/kernels:constant_op",
+        "@org_tensorflow//tensorflow/core/kernels:control_flow_ops",
+        "@org_tensorflow//tensorflow/core/kernels:conv_ops",
+        "@org_tensorflow//tensorflow/core/kernels:cwise_op",
+        "@org_tensorflow//tensorflow/core/kernels:depthwise_conv_op",
+        "@org_tensorflow//tensorflow/core/kernels:fused_batch_norm_op",
+        "@org_tensorflow//tensorflow/core/kernels:gather_op",
+        "@org_tensorflow//tensorflow/core/kernels:identity_op",
+        "@org_tensorflow//tensorflow/core/kernels:matmul_op",
+        "@org_tensorflow//tensorflow/core/kernels:non_max_suppression_op",
+        "@org_tensorflow//tensorflow/core/kernels:pack_op",
+        "@org_tensorflow//tensorflow/core/kernels:reduction_ops",
+        "@org_tensorflow//tensorflow/core/kernels:relu_op",
+        "@org_tensorflow//tensorflow/core/kernels:reshape_op",
+        "@org_tensorflow//tensorflow/core/kernels:resize_bilinear_op",
+        "@org_tensorflow//tensorflow/core/kernels:sequence_ops",
+        "@org_tensorflow//tensorflow/core/kernels:shape_ops",
+        "@org_tensorflow//tensorflow/core/kernels:slice_op",
+        "@org_tensorflow//tensorflow/core/kernels:split_op",
+        "@org_tensorflow//tensorflow/core/kernels:tensor_array_ops",
+        "@org_tensorflow//tensorflow/core/kernels:tile_ops",
+        "@org_tensorflow//tensorflow/core/kernels:topk_op",
+        "@org_tensorflow//tensorflow/core/kernels:transpose_op",
+        "@org_tensorflow//tensorflow/core/kernels:unpack_op",
+    ],
+)
+
+cc_binary(
+    name = "object_detection_tensorflow",
+    deps = [
+        ":object_detection_tensorflow_deps",
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/graphs/object_detection:desktop_tensorflow_calculators",
+    ],
+)
+
+cc_binary(
+    name = "object_detection_tflite",
+    deps = [
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/graphs/object_detection:desktop_tflite_calculators",
+    ],
+)

mediapipe/examples/desktop/simple_run_graph_main.cc

@@ -0,0 +1,69 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// A simple main function to run a MediaPipe graph.
+
+#include "absl/strings/str_split.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/commandlineflags.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/map_util.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+
+DEFINE_string(
+    calculator_graph_config_file, "",
+    "Name of file containing text format CalculatorGraphConfig proto.");
+
+DEFINE_string(input_side_packets, "",
+              "Comma-separated list of key=value pairs specifying side packets "
+              "for the CalculatorGraph. All values will be treated as the "
+              "string type even if they represent doubles, floats, etc.");
+
+::mediapipe::Status RunMediaPipeGraph() {
+  std::string calculator_graph_config_contents;
+  RETURN_IF_ERROR(mediapipe::file::GetContents(
+      FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
+  LOG(INFO) << "Get calculator graph config contents: "
+            << calculator_graph_config_contents;
+  mediapipe::CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
+          calculator_graph_config_contents);
+  std::map<std::string, ::mediapipe::Packet> input_side_packets;
+  std::vector<std::string> kv_pairs =
+      absl::StrSplit(FLAGS_input_side_packets, ',');
+  for (const std::string& kv_pair : kv_pairs) {
+    std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
+    RET_CHECK(name_and_value.size() == 2);
+    RET_CHECK(!::mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
+    input_side_packets[name_and_value[0]] =
+        ::mediapipe::MakePacket<std::string>(name_and_value[1]);
+  }
+  LOG(INFO) << "Initialize the calculator graph.";
+  mediapipe::CalculatorGraph graph;
+  RETURN_IF_ERROR(graph.Initialize(config, input_side_packets));
+  LOG(INFO) << "Start running the calculator graph.";
+  return graph.Run();
+}
+
+int main(int argc, char** argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  ::mediapipe::Status run_status = RunMediaPipeGraph();
+  if (!run_status.ok()) {
+    LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+  } else {
+    LOG(INFO) << "Success!";
+  }
+  return 0;
+}