Project import generated by Copybara.

GitOrigin-RevId: b137378673f7d66d41bcd46e4fc3a0d9ef254894

mediapipe/examples/desktop/BUILD

@@ -27,7 +27,9 @@ cc_library(
         "//mediapipe/framework/port:file_helpers",
         "//mediapipe/framework/port:map_util",
         "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
         "@com_google_absl//absl/strings",
     ],
 )

mediapipe/examples/desktop/simple_run_graph_main.cc

@@ -13,14 +13,23 @@
 // limitations under the License.
 //
 // A simple main function to run a MediaPipe graph.
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
 
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_split.h"
+#include "absl/strings/string_view.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/file_helpers.h"
 #include "mediapipe/framework/port/map_util.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/statusor.h"
 
 DEFINE_string(
     calculator_graph_config_file, "",
@@ -31,14 +40,72 @@ DEFINE_string(input_side_packets, "",
               "for the CalculatorGraph. All values will be treated as the "
               "string type even if they represent doubles, floats, etc.");
 
+// Local file output flags.
+// Output stream
+DEFINE_string(output_stream, "",
+              "The output stream to output to the local file in csv format.");
+DEFINE_string(output_stream_file, "",
+              "The name of the local file to output all packets sent to "
+              "the stream specified with --output_stream. ");
+DEFINE_bool(strip_timestamps, false,
+            "If true, only the packet contents (without timestamps) will be "
+            "written into the local file.");
+// Output side packets
+DEFINE_string(output_side_packets, "",
+              "A CSV of output side packets to output to local file.");
+DEFINE_string(output_side_packets_file, "",
+              "The name of the local file to output all side packets specified "
+              "with --output_side_packets. ");
+
+::mediapipe::Status OutputStreamToLocalFile(
+    ::mediapipe::OutputStreamPoller& poller) {
+  std::ofstream file;
+  file.open(FLAGS_output_stream_file);
+  ::mediapipe::Packet packet;
+  while (poller.Next(&packet)) {
+    std::string output_data;
+    if (!FLAGS_strip_timestamps) {
+      absl::StrAppend(&output_data, packet.Timestamp().Value(), ",");
+    }
+    absl::StrAppend(&output_data, packet.Get<std::string>(), "\n");
+    file << output_data;
+  }
+  file.close();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status OutputSidePacketsToLocalFile(
+    ::mediapipe::CalculatorGraph& graph) {
+  if (!FLAGS_output_side_packets.empty() &&
+      !FLAGS_output_side_packets_file.empty()) {
+    std::ofstream file;
+    file.open(FLAGS_output_side_packets_file);
+    std::vector<std::string> side_packet_names =
+        absl::StrSplit(FLAGS_output_side_packets, ',');
+    for (const std::string& side_packet_name : side_packet_names) {
+      ASSIGN_OR_RETURN(auto status_or_packet,
+                       graph.GetOutputSidePacket(side_packet_name));
+      file << absl::StrCat(side_packet_name, ":",
+                           status_or_packet.Get<std::string>(), "\n");
+    }
+    file.close();
+  } else {
+    RET_CHECK(FLAGS_output_side_packets.empty() &&
+              FLAGS_output_side_packets_file.empty())
+        << "--output_side_packets and --output_side_packets_file should be "
+           "specified in pair.";
+  }
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status RunMPPGraph() {
   std::string calculator_graph_config_contents;
-  MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
+  MP_RETURN_IF_ERROR(::mediapipe::file::GetContents(
       FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
   LOG(INFO) << "Get calculator graph config contents: "
             << calculator_graph_config_contents;
-  mediapipe::CalculatorGraphConfig config =
-      mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
+  ::mediapipe::CalculatorGraphConfig config =
+      ::mediapipe::ParseTextProtoOrDie<::mediapipe::CalculatorGraphConfig>(
           calculator_graph_config_contents);
   std::map<std::string, ::mediapipe::Packet> input_side_packets;
   std::vector<std::string> kv_pairs =
@@ -51,10 +118,23 @@ DEFINE_string(input_side_packets, "",
         ::mediapipe::MakePacket<std::string>(name_and_value[1]);
   }
   LOG(INFO) << "Initialize the calculator graph.";
-  mediapipe::CalculatorGraph graph;
+  ::mediapipe::CalculatorGraph graph;
   MP_RETURN_IF_ERROR(graph.Initialize(config, input_side_packets));
-  LOG(INFO) << "Start running the calculator graph.";
-  return graph.Run();
+  if (!FLAGS_output_stream.empty() && !FLAGS_output_stream_file.empty()) {
+    ASSIGN_OR_RETURN(auto poller,
+                     graph.AddOutputStreamPoller(FLAGS_output_stream));
+    LOG(INFO) << "Start running the calculator graph.";
+    MP_RETURN_IF_ERROR(graph.StartRun({}));
+    MP_RETURN_IF_ERROR(OutputStreamToLocalFile(poller));
+  } else {
+    RET_CHECK(FLAGS_output_stream.empty() && FLAGS_output_stream_file.empty())
+        << "--output_stream and --output_stream_file should be specified in "
+           "pair.";
+    LOG(INFO) << "Start running the calculator graph.";
+    MP_RETURN_IF_ERROR(graph.StartRun({}));
+  }
+  MP_RETURN_IF_ERROR(graph.WaitUntilDone());
+  return OutputSidePacketsToLocalFile(graph);
 }
 
 int main(int argc, char** argv) {

mediapipe/examples/desktop/youtube8m/BUILD

@@ -33,3 +33,14 @@ cc_binary(
         "@org_tensorflow//tensorflow/core:direct_session",
     ],
 )
+
+cc_binary(
+    name = "model_inference",
+    deps = [
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/graphs/youtube8m:yt8m_inference_calculators_deps",
+        # TODO: Figure out the minimum set of the kernels needed by this example.
+        "@org_tensorflow//tensorflow/core:all_kernels",
+        "@org_tensorflow//tensorflow/core:direct_session",
+    ],
+)

mediapipe/examples/desktop/youtube8m/README.md

@@ -1,13 +1,13 @@
 ### Steps to run the YouTube-8M feature extraction graph
 
-1.  Checkout the mediapipe repository
+1.  Checkout the mediapipe repository.
 
     ```bash
     git clone https://github.com/google/mediapipe.git
     cd mediapipe
     ```
 
-2.  Download the PCA and model data
+2.  Download the PCA and model data.
 
     ```bash
     mkdir /tmp/mediapipe
@@ -20,7 +20,7 @@
     tar -xvf /tmp/mediapipe/inception-2015-12-05.tgz
     ```
 
-3.  Get the VGGish frozen graph
+3.  Get the VGGish frozen graph.
 
     Note: To run step 3 and step 4, you must have Python 2.7 or 3.5+ installed
     with the TensorFlow 1.14+ package installed.
@@ -31,26 +31,114 @@
     python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph
     ```
 
-4.  Generate a MediaSequence metadata from the input video
+4.  Generate a MediaSequence metadata from the input video.
 
     Note: the output file is /tmp/mediapipe/metadata.tfrecord
 
     ```bash
+    # change clip_end_time_sec to match the length of your video.
     python -m mediapipe.examples.desktop.youtube8m.generate_input_sequence_example \
       --path_to_input_video=/absolute/path/to/the/local/video/file \
-      --clip_start_time_sec=0 \
-      --clip_end_time_sec=10
+      --clip_end_time_sec=120
     ```
 
-5.  Run the MediaPipe binary to extract the features
+5.  Run the MediaPipe binary to extract the features.
 
     ```bash
     bazel build -c opt \
       --define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \
       mediapipe/examples/desktop/youtube8m:extract_yt8m_features
 
-    ./bazel-bin/mediapipe/examples/desktop/youtube8m/extract_yt8m_features \
+    GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/extract_yt8m_features \
       --calculator_graph_config_file=mediapipe/graphs/youtube8m/feature_extraction.pbtxt \
       --input_side_packets=input_sequence_example=/tmp/mediapipe/metadata.tfrecord  \
       --output_side_packets=output_sequence_example=/tmp/mediapipe/output.tfrecord
     ```
+
+### Steps to run the YouTube-8M inference graph with the YT8M dataset
+
+1.  Download the YT8M dataset
+
+  For example, download one shard of the training data:
+
+  ```bash
+  curl http://us.data.yt8m.org/2/frame/train/trainpj.tfrecord --output /tmp/mediapipe/trainpj.tfrecord
+  ```
+
+2.  Copy the baseline model [(model card)](https://drive.google.com/file/d/1xTCi9-Nm9dt2KIk8WR0dDFrIssWawyXy/view) to local.
+
+  ```bash
+  curl -o /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz data.yt8m.org/models/baseline/saved_model.tar.gz
+
+  tar -xvf /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz -C /tmp/mediapipe
+  ```
+
+3.  Build and run the inference binary.
+
+  ```bash
+  bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
+    mediapipe/examples/desktop/youtube8m:model_inference
+
+  GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/model_inference \
+    --calculator_graph_config_file=mediapipe/graphs/youtube8m/yt8m_dataset_model_inference.pbtxt \
+    --input_side_packets=tfrecord_path=/tmp/mediapipe/trainpj.tfrecord,record_index=0,desired_segment_size=5 \
+    --output_stream=annotation_summary \
+    --output_stream_file=/tmp/summary \
+    --output_side_packets=yt8m_id \
+    --output_side_packets_file=/tmp/yt8m_id
+  ```
+
+### Steps to run the YouTube-8M model inference graph with Web Interface
+
+1.  Copy the baseline model [(model card)](https://drive.google.com/file/d/1xTCi9-Nm9dt2KIk8WR0dDFrIssWawyXy/view) to local.
+
+
+  ```bash
+  curl -o /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz data.yt8m.org/models/baseline/saved_model.tar.gz
+
+  tar -xvf /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz -C /tmp/mediapipe
+  ```
+
+2.  Build the inference binary.
+
+  ```bash
+  bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
+    mediapipe/examples/desktop/youtube8m:model_inference
+  ```
+
+3.  Run the python web server.
+
+  Note: pip install absl-py
+
+  ```bash
+  python mediapipe/examples/desktop/youtube8m/viewer/server.py --root `pwd`
+  ```
+
+  Navigate to localhost:8008 in a web browser.
+
+### Steps to run the YouTube-8M model inference graph with a local video
+
+1.  Make sure you have the output tfrecord from the feature extraction pipeline.
+
+2.  Copy the baseline model [(model card)](https://drive.google.com/file/d/1xTCi9-Nm9dt2KIk8WR0dDFrIssWawyXy/view) to local.
+
+    ```bash
+    curl -o /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz data.yt8m.org/models/baseline/saved_model.tar.gz
+
+    tar -xvf /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz -C /tmp/mediapipe
+    ```
+
+3.  Build and run the inference binary.
+
+  ```bash
+  bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
+    mediapipe/examples/desktop/youtube8m:model_inference
+
+  # segment_size is the number of seconds window of frames.
+  # overlap is the number of seconds adjacent segments share.
+  GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/model_inference \
+    --calculator_graph_config_file=mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt \
+    --input_side_packets=input_sequence_example_path=/tmp/mediapipe/output.tfrecord,input_video_path=/absolute/path/to/the/local/video/file,output_video_path=/tmp/mediapipe/annotated_video.mp4,segment_size=5,overlap=4
+  ```
+
+4.  View the annotated video.

mediapipe/examples/desktop/youtube8m/viewer/server.py

@@ -0,0 +1,262 @@
+"""Server for YouTube8M Model Inference Demo.
+
+Serves up both the static files for the website and provides a service that
+fetches the video id and timestamp based labels for a video analyzed in a
+tfrecord files.
+
+"""
+from __future__ import print_function
+import json
+import os
+import re
+import socket
+import subprocess
+import sys
+
+from absl import app
+from absl import flags
+import http.client
+import http.server
+from six.moves.urllib import parse
+
+FLAGS = flags.FLAGS
+flags.DEFINE_bool("show_label_at_center", False,
+                  "Show labels at the center of the segment.")
+flags.DEFINE_integer("port", 8008, "Port that the API is served over.")
+flags.DEFINE_string("tmp_dir", "/tmp/mediapipe",
+                    "Temporary asset storage location.")
+flags.DEFINE_string("root", "", "MediaPipe root directory.")
+# binary, pbtxt, label_map paths are relative to 'root' path
+flags.DEFINE_string(
+    "binary",
+    "bazel-bin/mediapipe/examples/desktop/youtube8m/model_inference",
+    "Inference binary location.")
+flags.DEFINE_string(
+    "pbtxt",
+    "mediapipe/graphs/youtube8m/yt8m_dataset_model_inference.pbtxt",
+    "Default pbtxt graph file.")
+flags.DEFINE_string("label_map", "mediapipe/graphs/youtube8m/label_map.txt",
+                    "Default label map text file.")
+
+
+class HTTPServerV6(http.server.HTTPServer):
+  address_family = socket.AF_INET6
+
+
+class Youtube8MRequestHandler(http.server.SimpleHTTPRequestHandler):
+  """Static file server with /healthz support."""
+
+  def do_GET(self):
+    if self.path.startswith("/healthz"):
+      self.send_response(200)
+      self.send_header("Content-type", "text/plain")
+      self.send_header("Content-length", 2)
+      self.end_headers()
+      self.wfile.write("ok")
+    if self.path.startswith("/video"):
+      parsed_params = parse.urlparse(self.path)
+      url_params = parse.parse_qs(parsed_params.query)
+
+      tfrecord_path = ""
+      segment_size = 5
+
+      print(url_params)
+      if "file" in url_params:
+        tfrecord_path = url_params["file"][0]
+      if "segments" in url_params:
+        segment_size = int(url_params["segments"][0])
+
+      self.fetch(tfrecord_path, segment_size)
+
+    else:
+      if self.path == "/":
+        self.path = "/index.html"
+      # Default to serve up a local file
+      self.path = "/static" + self.path
+      http.server.SimpleHTTPRequestHandler.do_GET(self)
+
+  def report_error(self, msg):
+    """Simplifies sending out a string as a 500 http response."""
+    self.send_response(500)
+    self.send_header("Content-type", "text/plain")
+    self.end_headers()
+    if sys.version_info[0] < 3:
+      self.wfile.write(str(msg).encode("utf-8"))
+    else:
+      self.wfile.write(bytes(msg, "utf-8"))
+
+  def report_missing_files(self, files):
+    """Sends out 500 response with missing files."""
+    accumulate = ""
+    for file_path in files:
+      if not os.path.exists(file_path):
+        accumulate = "%s '%s'" % (accumulate, file_path)
+
+    if accumulate:
+      self.report_error("Could not find:%s" % accumulate)
+      return True
+
+    return False
+
+  def fetch(self, path, segment_size):
+    """Returns the video id and labels for a tfrecord at a provided index."""
+
+    print("Received request.  File=", path, "Segment Size =", segment_size)
+
+    if (self.report_missing_files([
+        "%s/%s" % (FLAGS.root, FLAGS.pbtxt),
+        "%s/%s" % (FLAGS.root, FLAGS.binary),
+        "%s/%s" % (FLAGS.root, FLAGS.label_map)
+    ])):
+      return
+
+    # Parse the youtube video id off the end of the link or as a standalone id.
+    filename_match = re.match(
+        "(?:.*youtube.*v=)?([a-zA-Z-0-9_]{2})([a-zA-Z-0-9_]+)", path)
+    tfrecord_url = filename_match.expand(r"data.yt8m.org/2/j/r/\1/\1\2.js")
+
+    print("Trying to get tfrecord via", tfrecord_url)
+
+    connection = http.client.HTTPConnection("data.yt8m.org")
+    connection.request("GET", tfrecord_url)
+    response = connection.getresponse()
+
+    response_object = json.loads(response.read())
+    filename = response_object["filename_raw"]
+    index = response_object["index"]
+
+    print("TFRecord discovered: ", filename, ", index", index)
+
+    output_file = r"%s/%s" % (FLAGS.tmp_dir, filename)
+    tfrecord_url = r"http://us.data.yt8m.org/2/frame/train/%s" % filename
+
+    connection = http.client.HTTPConnection("us.data.yt8m.org")
+    connection.request("HEAD",
+                       filename_match.expand(r"/2/frame/train/%s" % filename))
+    response = connection.getresponse()
+    if response.getheader("Content-Type") != "application/octet-stream":
+      self.report_error("Filename '%s' is invalid." % path)
+
+    print(output_file, "exists on yt8m.org. Did we fetch this before?")
+
+    if not os.path.exists(output_file):
+      print(output_file, "doesn't exist locally, download it now.")
+      return_code = subprocess.call(
+          ["curl", "--output", output_file, tfrecord_url],
+          stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+      if return_code:
+        self.report_error("Could not retrieve contents from %s" % tfrecord_url)
+        return
+    else:
+      print(output_file, "exist locally, reuse it.")
+
+    print("Run the graph...")
+    process = subprocess.Popen([
+        "%s/%s" % (FLAGS.root, FLAGS.binary),
+        "--calculator_graph_config_file=%s/%s" % (FLAGS.root, FLAGS.pbtxt),
+        "--input_side_packets=tfrecord_path=%s" % output_file +
+        ",record_index=%d" % index + ",desired_segment_size=%d" % segment_size,
+        "--output_stream=annotation_summary",
+        "--output_stream_file=%s/labels" % FLAGS.tmp_dir,
+        "--output_side_packets=yt8m_id",
+        "--output_side_packets_file=%s/yt8m_id" % FLAGS.tmp_dir
+    ],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout_str, stderr_str = process.communicate()
+    process.wait()
+
+    if stderr_str and "success" not in str(stderr_str).lower():
+      self.report_error("Error executing server binary: \n%s" % stderr_str)
+      return
+
+    f = open("%s/yt8m_id" % FLAGS.tmp_dir, "r")
+    contents = f.read()
+    print("yt8m_id is", contents[-5:-1])
+
+    curl_arg = "data.yt8m.org/2/j/i/%s/%s.js" % (contents[-5:-3],
+                                                 contents[-5:-1])
+    print("Grab labels from", curl_arg)
+    process = subprocess.Popen(["curl", curl_arg],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout = process.communicate()
+    process.wait()
+
+    stdout_str = stdout[0].decode("utf-8")
+
+    match = re.match(""".+"([^"]+)"[^"]+""", stdout_str)
+    final_results = {
+        "video_id": match.group(1),
+        "link": "https://www.youtube.com/watch?v=%s" % match.group(1),
+        "entries": []
+    }
+    f = open("%s/labels" % FLAGS.tmp_dir, "r")
+    lines = f.readlines()
+    show_at_center = FLAGS.show_label_at_center
+
+    print("%s/labels" % FLAGS.tmp_dir, "holds", len(lines), "entries")
+    for line in lines:
+      entry = {"labels": []}
+      final_results["entries"].append(entry)
+      first = True
+      for column in line.split(","):
+        if first:
+          subtract = segment_size / 2.0 if show_at_center else 0.0
+          entry["time"] = float(int(column)) / 1000000.0 - subtract
+          first = False
+        else:
+          label_score = re.match("(.+):([0-9.]+).*", column)
+          if label_score:
+            score = float(label_score.group(2))
+            entry["labels"].append({
+                "label": label_score.group(1),
+                "score": score
+            })
+          else:
+            print("empty score")
+
+    response_json = json.dumps(final_results, indent=2, separators=(",", ": "))
+    self.send_response(200)
+    self.send_header("Content-type", "application/json")
+    self.end_headers()
+    if sys.version_info[0] < 3:
+      self.wfile.write(str(response_json).encode("utf-8"))
+    else:
+      self.wfile.write(bytes(response_json, "utf-8"))
+
+
+def update_pbtxt():
+  """Update graph.pbtxt to use full path to label_map.txt."""
+  edited_line = ""
+  lines = []
+  with open("%s/%s" % (FLAGS.root, FLAGS.pbtxt), "r") as f:
+    lines = f.readlines()
+    for line in lines:
+      if "label_map_path" in line:
+        kv = line.split(":")
+        edited_line = kv[0] + (": \"%s/%s\"\n" % (FLAGS.root, FLAGS.label_map))
+  with open("%s/%s" % (FLAGS.root, FLAGS.pbtxt), "w") as f:
+    for line in lines:
+      if "label_map_path" in line:
+        f.write(edited_line)
+      else:
+        f.write(line)
+
+
+def main(unused_args):
+  dname = os.path.dirname(os.path.abspath(__file__))
+  os.chdir(dname)
+  if not FLAGS.root:
+    print("Must specify MediaPipe root directory: --root `pwd`")
+    return
+  update_pbtxt()
+  port = FLAGS.port
+  print("Listening on port %s" % port)  # pylint: disable=superfluous-parens
+  server = HTTPServerV6(("::", int(port)), Youtube8MRequestHandler)
+  server.serve_forever()
+
+
+if __name__ == "__main__":
+  app.run(main)

mediapipe/examples/desktop/youtube8m/viewer/static/index.html

@@ -0,0 +1,96 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <title>MediaPipe: YouTube8M Model Inference Demo</title>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+  <base href="/">
+  <script src="main.js"></script>
+  <link rel="stylesheet"
+        href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css"
+        integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T"
+        crossorigin="anonymous">
+</head>
+
+<body>
+  <div class="container-fluid">
+    <h2>
+      MediaPipe: YouTube8M Model Inference Demo
+    </h2>
+    <form id="form">
+      <div class="row">
+        <div class="card m-2" style="width: 640px;">
+          <div>
+            <div style="position:relative;">
+              <iframe id="ytplayer" style="display:none;" type="text/html" width="640" height="320"
+                      src="https://www.youtube.com/embed/M7lc1UVf-VE?enablejsapi=1" frameborder="0"
+                      enablejsapi="1"></iframe>
+              <div id="cover" class="bg-warning"
+                   style="width:640px; height:320px;">
+              </div>
+              <div id="spinner" class="bg-warning"
+                   style="display: none; width:640px; height:320px;">
+                  <div class="spinner-border" role="status"
+                       style="position:relative; left:300px; top:130px;">
+                    <span class="sr-only">Loading...</span>
+                  </div>
+              </div>
+            </div>
+          </div>
+          <div class="card-body shadow">
+            <div class="row mb-2">
+              <ul class="nav">
+                <li class="nav-item">
+                  <a class="nav-link"
+                     href="https://research.google.com/youtube8m/explore.html"
+                     target="_">Explore Videos</a>
+                </li>
+              </ul>
+            </div>
+              <div class="form-group">
+                <label for="file">YouTube video ID</label>
+                <input type="text" class="form-control" name="file" id="file"
+                       placeholder="Enter a YouTube link or a YouTube ID">
+                <small class="form-text text-muted">
+                  e.g., Both "https://youtube.com/watch?v=huGVGe3Afng" or "huGVGe3Afng" will work.
+                </small>
+              </div>
+              <div class="form-group">
+                <label id="segments_label" for="segments">Segment Size</label>
+                <input type="range" min="1" max="300" step="1" value="5"
+                       class="form-control-range" name="segments" id="segments">
+              </div>
+              <button type="submit" class="btn btn-primary">Submit</button>
+              <div id="error_msg" style="visibility:hidden;" class="alert alert-danger mt-2"
+                   role="alert"></div>
+          </div>
+        </div>
+
+        <div class="card m-2 shadow">
+          <div class="card-body">
+            <div class="form-group">
+              <label id="threshold_label" for="threshold">Score Threshold</label>
+              <input type="range" min="0" max="0.99" step="0.01" value="0.2"
+                     class="form-control-range" name="threshold" id="threshold">
+            </div>
+            <h5>
+              Labels
+            </h5>
+            <textarea id="feedback" style="height:320px; width:500px;"></textarea>
+          </div>
+        </div>
+      </div>
+    </form>
+  </div>
+</body>
+<script async src="https://www.youtube.com/iframe_api"></script>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"
+        integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1"
+        crossorigin="anonymous"></script>
+<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"
+        integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM"
+        crossorigin="anonymous"></script>
+</html>
+
+

mediapipe/examples/desktop/youtube8m/viewer/static/main.js

@@ -0,0 +1,217 @@
+/**
+ * @license
+ * Copyright 2019 The MediaPipe Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const STATE_PLAYER=0;
+const STATE_COVER=1;
+const STATE_SPINNER=2;
+
+/**
+* Looks up the value of a url parameter.
+*
+* @param {string} param The name of the parameter.
+* @return {?string} The parameter value or null if there is no such parameter.
+*/
+var getUrlParameter = function(param) {
+    const url = decodeURIComponent(window.location.search.substring(1));
+    const url_parts = url.split('&');
+    for (var i = 0; i < url_parts.length; i++) {
+        const param_name = url_parts[i].split(/=(.*)/);
+        if (param_name[0] === param) {
+            return param_name[1] === undefined ? null : param_name[1];
+        }
+    }
+};
+
+/**
+* Sets the fields in the form to match the values of the URL parameters.
+*/
+const updateFormFromURL = function() {
+  const form_elements = document.getElementById('form').elements;
+  const url = decodeURIComponent(window.location.search.substring(1));
+  const url_parts = url.split('&');
+  for (var i = 0; i < url_parts.length; i++) {
+    const p = url_parts[i].split(/=(.*)/);
+    if (p.length >= 2) {
+      if (form_elements[p[0]]) {
+        form_elements[p[0]].value = decodeURIComponent(p[1]);
+      }
+    }
+  }
+};
+
+let player = null;
+let intervalID = undefined;
+let entries = [];
+
+/**
+ * Constructs the embedded YouTube player.
+ */
+window.onYouTubeIframeAPIReady = () => {
+  player = new YT.Player('ytplayer', {
+    events: {
+      'onReady': onPlayerReady,
+      'onStateChange': onStateChange
+    }
+  });
+};
+
+
+/**
+ * Listens for YouTube video events. When video is playing, periodically checks
+ * the time signature and updates the feedback with labels. When video stops,
+ * shuts off interval timer to save cycles.
+ * @param {!Event} event YouTube API Event.
+ */
+function onStateChange(event) {
+  if (event.data === 1) {
+    // Youtube switched to playing.
+    intervalID = setInterval(function(){
+      const currentTime = player.getCurrentTime();
+      let winner = undefined;
+      let first = undefined;
+      for (entry of entries) {
+        if (!first) {
+          first = entry.labels;
+        }
+        if (entry.time < currentTime) {
+          winner = entry.labels;
+        } else {
+          break;
+        }
+      }
+      if (!winner) {
+        winner = first;
+      }
+      const threshold =
+          document.getElementById('form').elements['threshold'].value;
+      let message = "";
+      for (var label of winner) {
+        if (label.score >= threshold) {
+          message =  `${message}${label.label} (score: ${label.score})\n`;
+        }
+      }
+      $("textarea#feedback").val(message);
+    });
+  } else {
+    if (intervalID) {
+      clearInterval(intervalID);
+    }
+  }
+}
+
+/**
+ * Turns elements of the player on and off to reflect the state of the "app".
+ * @param {number} state One of STATE_COVER | STATE_SPINNER | STATE_PLAYER.
+ */
+function showState(state) {
+  switch(state) {
+    case STATE_COVER:
+      $('#cover').show();
+      $('#spinner').hide();
+      $('#ytplayer').hide();
+      break;
+    case STATE_SPINNER:
+      $('#cover').hide();
+      $('#spinner').show();
+      $('#ytplayer').hide();
+      break;
+    case STATE_PLAYER:
+    default:
+      $('#cover').hide();
+      $('#spinner').hide();
+      $('#ytplayer').show();
+      break;
+  }
+}
+
+/**
+ * Hide error field and clear its message.
+ */
+function hideError() {
+  $('#error_msg').css("visibility", "hidden").text('');
+}
+
+/**
+ * Set the error to visible and set its message.
+ * @param {string} msg Error message as a string.
+ */
+function showError(msg) {
+  $('#error_msg').css("visibility", "visible").text(msg);
+}
+
+/**
+ * Privides numeric feedback for the slider.
+ */
+function connectSlider() {
+  $('#threshold_label').text(
+      `Score Threshold (${$('#threshold')[0].value})`);
+  $('#threshold').on('input', () => {
+    $('#threshold_label').text(
+        `Score Threshold (${$('#threshold')[0].value})`);
+  });
+  $('#segments_label').text(
+      `Segment Size (${$('#segments')[0].value})`);
+  $('#segments').on('input', () => {
+    $('#segments_label').text(
+        `Segment Size (${$('#segments')[0].value})`);
+  });
+}
+
+/**
+ * Retrieve video information from backend.
+ * @param {string} filePath name of a tfrecord file.
+ * @param {number} segments desired number of segments (1-300)
+ */
+function fetchVideo(filePath, segments) {
+  const url = "/video?file=" + filePath + "&segments=" + segments;
+  $.ajax({
+    url: url,
+    success: function(result) {
+      const videoId = result["video_id"];
+      player.loadVideoById(videoId);
+      entries = result['entries'];
+      showState(STATE_PLAYER);
+    },
+    error: (err) => {
+      showState(STATE_COVER);
+      console.log(err);
+      showError(err.responseText);
+    },
+    datatype: "json"
+  });
+}
+
+/**
+ * Called when the embedded YouTube player has finished loading. It loads the
+ * requested video into the player and calls the golden6_viewer API to retrieve
+ * the frame-level data for that video.
+ */
+function onPlayerReady() {
+  const filePath = getUrlParameter('file') || "";
+  const segments = parseInt(getUrlParameter('segments')) || 0;
+
+  updateFormFromURL();
+  hideError();
+  connectSlider();
+
+  if (!filePath) {
+    return;
+  }
+
+  showState(STATE_SPINNER);
+  fetchVideo(filePath, segments);
+}