Project import generated by Copybara.

GitOrigin-RevId: 852dfb05d450167899c0dd5ef7c45622a12e865b

README.md

@@ -1,7 +1,7 @@
 ![MediaPipe](mediapipe/docs/images/mediapipe_small.png?raw=true "MediaPipe logo")
 =======================================================================
 
-[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.
+[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data), cross platform (i.e Android, iOS, web, edge devices) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.
 
 ![Real-time Face Detection](mediapipe/docs/images/realtime_face_detection.gif)
 
@@ -9,17 +9,17 @@
 
 ## ML Solutions in MediaPipe
 
-* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md)
+* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
 * [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
-* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md)
-* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
+* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
+* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
 * [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
 * [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
 * [AutoFlip](mediapipe/docs/autoflip.md)
 
-![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
-![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
 ![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
+![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
+![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
 ![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
 ![object_tracking](mediapipe/docs/images/mobile/object_tracking_android_gpu_small.gif)
 
@@ -29,6 +29,8 @@ Follow these [instructions](mediapipe/docs/install.md).
 ## Getting started
 See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).
 
+Check out some web demos [[Edge detection]](https://viz.mediapipe.dev/runner/demos/edge_detection/edge_detection.html) [[Face detection]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html) [[Hand Tracking]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
+
 ## Documentation
 [MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev)
 
@@ -37,10 +39,12 @@ Check out the [Examples page](https://mediapipe.readthedocs.io/en/latest/example
 ## Visualizing MediaPipe graphs
 A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.dev/). Please also see instructions [here](mediapipe/docs/visualizer.md).
 
-## Community forum
-*  [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
+## Videos
+*  [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)
 
 ## Publications
+* [Google Developer Blog: MediaPipe on the Web](https://mediapipe.page.link/webdevblog)
+* [Google Developer Blog: Object Detection and Tracking using MediaPipe](https://mediapipe.page.link/objecttrackingblog)
 * [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
 * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
 
@@ -55,6 +59,9 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
 * [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan
 * [Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
 
+## Community forum
+*  [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
+
 ## Alpha Disclaimer
 MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.
 

WORKSPACE

@@ -78,6 +78,14 @@ http_archive(
     ],
 )
 
+# easyexif
+http_archive(
+    name = "easyexif",
+    url = "https://github.com/mayanklahiri/easyexif/archive/master.zip",
+    strip_prefix = "easyexif-master",
+    build_file = "@//third_party:easyexif.BUILD",
+)
+
 # libyuv
 http_archive(
     name = "libyuv",

mediapipe/calculators/core/BUILD

@@ -86,6 +86,15 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "constant_side_packet_calculator_proto",
+    srcs = ["constant_side_packet_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
 proto_library(
     name = "clip_vector_size_calculator_proto",
     srcs = ["clip_vector_size_calculator.proto"],
@@ -173,6 +182,14 @@ mediapipe_cc_proto_library(
     deps = [":gate_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "constant_side_packet_calculator_cc_proto",
+    srcs = ["constant_side_packet_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":constant_side_packet_calculator_proto"],
+)
+
 cc_library(
     name = "add_header_calculator",
     srcs = ["add_header_calculator.cc"],
@@ -960,3 +977,30 @@ cc_test(
         "@com_google_absl//absl/memory",
     ],
 )
+
+cc_library(
+    name = "constant_side_packet_calculator",
+    srcs = ["constant_side_packet_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":constant_side_packet_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "constant_side_packet_calculator_test",
+    srcs = ["constant_side_packet_calculator_test.cc"],
+    deps = [
+        ":constant_side_packet_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)

mediapipe/calculators/core/clip_vector_size_calculator.h

@@ -38,6 +38,8 @@ namespace mediapipe {
 //     }
 //   }
 // }
+// Optionally, you can pass in a side packet that will override `max_vec_size`
+// that is specified in the options.
 template <typename T>
 class ClipVectorSizeCalculator : public CalculatorBase {
  public:
@@ -53,6 +55,10 @@ class ClipVectorSizeCalculator : public CalculatorBase {
 
     cc->Inputs().Index(0).Set<std::vector<T>>();
     cc->Outputs().Index(0).Set<std::vector<T>>();
+    // Optional input side packet that determines `max_vec_size`.
+    if (cc->InputSidePackets().NumEntries() > 0) {
+      cc->InputSidePackets().Index(0).Set<int>();
+    }
 
     return ::mediapipe::OkStatus();
   }
@@ -61,6 +67,11 @@ class ClipVectorSizeCalculator : public CalculatorBase {
     cc->SetOffset(TimestampDiff(0));
     max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
                         .max_vec_size();
+    // Override `max_vec_size` if passed as side packet.
+    if (cc->InputSidePackets().NumEntries() > 0 &&
+        !cc->InputSidePackets().Index(0).IsEmpty()) {
+      max_vec_size_ = cc->InputSidePackets().Index(0).Get<int>();
+    }
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/core/clip_vector_size_calculator_test.cc

@@ -176,4 +176,31 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
   }
 }
 
+TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "TestClipIntVectorSizeCalculator"
+        input_stream: "input_vector"
+        input_side_packet: "max_vec_size"
+        output_stream: "output_vector"
+        options {
+          [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  // This should override the default of 1 set in the options.
+  runner.MutableSidePackets()->Index(0) = Adopt(new int(2));
+  std::vector<int> input = {0, 1, 2, 3};
+  AddInputVector(input, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
+  EXPECT_EQ(2, output.size());
+  std::vector<int> expected_vector = {0, 1};
+  EXPECT_EQ(expected_vector, output);
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -35,6 +35,16 @@ namespace mediapipe {
 typedef ConcatenateVectorCalculator<float> ConcatenateFloatVectorCalculator;
 REGISTER_CALCULATOR(ConcatenateFloatVectorCalculator);
 
+// Example config:
+// node {
+//   calculator: "ConcatenateInt32VectorCalculator"
+//   input_stream: "int32_vector_1"
+//   input_stream: "int32_vector_2"
+//   output_stream: "concatenated_int32_vector"
+// }
+typedef ConcatenateVectorCalculator<int32> ConcatenateInt32VectorCalculator;
+REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
+
 // Example config:
 // node {
 //   calculator: "ConcatenateTfLiteTensorVectorCalculator"

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -0,0 +1,116 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "mediapipe/calculators/core/constant_side_packet_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Generates an output side packet or multiple output side packets according to
+// the specified options.
+//
+// Example configs:
+// node {
+//   calculator: "ConstantSidePacketCalculator"
+//   output_side_packet: "PACKET:packet"
+//   options: {
+//     [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+//       packet { int_value: 2 }
+//     }
+//   }
+// }
+//
+// node {
+//   calculator: "ConstantSidePacketCalculator"
+//   output_side_packet: "PACKET:0:int_packet"
+//   output_side_packet: "PACKET:1:bool_packet"
+//   options: {
+//     [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+//       packet { int_value: 2 }
+//       packet { bool_value: true }
+//     }
+//   }
+// }
+class ConstantSidePacketCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    const auto& options = cc->Options().GetExtension(
+        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+    RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag),
+                 options.packet_size())
+        << "Number of output side packets has to be same as number of packets "
+           "configured in options.";
+
+    int index = 0;
+    for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
+         id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {
+      const auto& packet_options = options.packet(index);
+      auto& packet = cc->OutputSidePackets().Get(id);
+      if (packet_options.has_int_value()) {
+        packet.Set<int>();
+      } else if (packet_options.has_float_value()) {
+        packet.Set<float>();
+      } else if (packet_options.has_bool_value()) {
+        packet.Set<bool>();
+      } else if (packet_options.has_string_value()) {
+        packet.Set<std::string>();
+      } else {
+        return ::mediapipe::InvalidArgumentError(
+            "None of supported values were specified in options.");
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    const auto& options = cc->Options().GetExtension(
+        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+    int index = 0;
+    for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
+         id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {
+      auto& packet = cc->OutputSidePackets().Get(id);
+      const auto& packet_options = options.packet(index);
+      if (packet_options.has_int_value()) {
+        packet.Set(MakePacket<int>(packet_options.int_value()));
+      } else if (packet_options.has_float_value()) {
+        packet.Set(MakePacket<float>(packet_options.float_value()));
+      } else if (packet_options.has_bool_value()) {
+        packet.Set(MakePacket<bool>(packet_options.bool_value()));
+      } else if (packet_options.has_string_value()) {
+        packet.Set(MakePacket<std::string>(packet_options.string_value()));
+      } else {
+        return ::mediapipe::InvalidArgumentError(
+            "None of supported values were specified in options.");
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  static constexpr const char* kPacketTag = "PACKET";
+};
+
+REGISTER_CALCULATOR(ConstantSidePacketCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/constant_side_packet_calculator.proto

@@ -0,0 +1,36 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message ConstantSidePacketCalculatorOptions {
+  extend CalculatorOptions {
+    optional ConstantSidePacketCalculatorOptions ext = 291214597;
+  }
+
+  message ConstantSidePacket {
+    oneof value {
+      int32 int_value = 1;
+      float float_value = 2;
+      bool bool_value = 3;
+      string string_value = 4;
+    }
+  }
+
+  repeated ConstantSidePacket packet = 1;
+}

mediapipe/calculators/core/constant_side_packet_calculator_test.cc

@@ -0,0 +1,196 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+template <typename T>
+void DoTestSingleSidePacket(absl::string_view packet_spec,
+                            const T& expected_value) {
+  static constexpr absl::string_view graph_config_template = R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:packet"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet $0
+            }
+          }
+        }
+      )";
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          absl::Substitute(graph_config_template, packet_spec));
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  MP_ASSERT_OK(graph.GetOutputSidePacket("packet"));
+  auto actual_value =
+      graph.GetOutputSidePacket("packet").ValueOrDie().template Get<T>();
+  EXPECT_EQ(actual_value, expected_value);
+}
+
+TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
+  DoTestSingleSidePacket("{ int_value: 2 }", 2);
+  DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
+  DoTestSingleSidePacket("{ bool_value: true }", true);
+  DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
+}
+
+TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:0:int_packet"
+          output_side_packet: "PACKET:1:float_packet"
+          output_side_packet: "PACKET:2:bool_packet"
+          output_side_packet: "PACKET:3:string_packet"
+          output_side_packet: "PACKET:4:another_string_packet"
+          output_side_packet: "PACKET:5:another_int_packet"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet { int_value: 256 }
+              packet { float_value: 0.5f }
+              packet { bool_value: false }
+              packet { string_value: "string" }
+              packet { string_value: "another string" }
+              packet { int_value: 128 }
+            }
+          }
+        }
+      )");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("int_packet").ValueOrDie().Get<int>(),
+            256);
+  MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("float_packet").ValueOrDie().Get<float>(),
+            0.5f);
+  MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet"));
+  EXPECT_FALSE(
+      graph.GetOutputSidePacket("bool_packet").ValueOrDie().Get<bool>());
+  MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("string_packet")
+                .ValueOrDie()
+                .Get<std::string>(),
+            "string");
+  MP_ASSERT_OK(graph.GetOutputSidePacket("another_string_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("another_string_packet")
+                .ValueOrDie()
+                .Get<std::string>(),
+            "another string");
+  MP_ASSERT_OK(graph.GetOutputSidePacket("another_int_packet"));
+  EXPECT_EQ(
+      graph.GetOutputSidePacket("another_int_packet").ValueOrDie().Get<int>(),
+      128);
+}
+
+TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:0:int_packet"
+          output_side_packet: "no_tag0"
+          output_side_packet: "PACKET:1:float_packet"
+          output_side_packet: "INCORRECT_TAG:0:name1"
+          output_side_packet: "PACKET:2:bool_packet"
+          output_side_packet: "PACKET:3:string_packet"
+          output_side_packet: "no_tag2"
+          output_side_packet: "INCORRECT_TAG:1:name2"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet { int_value: 256 }
+              packet { float_value: 0.5f }
+              packet { bool_value: false }
+              packet { string_value: "string" }
+            }
+          }
+        }
+      )");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("int_packet").ValueOrDie().Get<int>(),
+            256);
+  MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("float_packet").ValueOrDie().Get<float>(),
+            0.5f);
+  MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet"));
+  EXPECT_FALSE(
+      graph.GetOutputSidePacket("bool_packet").ValueOrDie().Get<bool>());
+  MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("string_packet")
+                .ValueOrDie()
+                .Get<std::string>(),
+            "string");
+}
+
+TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:int_packet"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet { int_value: 256 }
+              packet { float_value: 0.5f }
+            }
+          }
+        }
+      )");
+  CalculatorGraph graph;
+  EXPECT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:0:int_packet"
+          output_side_packet: "PACKET:1:float_packet"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet { int_value: 256 }
+            }
+          }
+        }
+      )");
+  CalculatorGraph graph;
+  EXPECT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -17,6 +17,12 @@
 #include <memory>
 
 namespace {
+// Reflect an integer against the lower and upper bound of an interval.
+int64 ReflectBetween(int64 ts, int64 ts_min, int64 ts_max) {
+  if (ts < ts_min) return 2 * ts_min - ts - 1;
+  if (ts >= ts_max) return 2 * ts_max - ts - 1;
+  return ts;
+}
 
 // Creates a secure random number generator for use in ProcessWithJitter.
 // If no secure random number generator can be constructed, the jitter
@@ -82,6 +88,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
 
   flush_last_packet_ = resampler_options.flush_last_packet();
   jitter_ = resampler_options.jitter();
+  jitter_with_reflection_ = resampler_options.jitter_with_reflection();
 
   input_data_id_ = cc->Inputs().GetId("DATA", 0);
   if (!input_data_id_.IsValid()) {
@@ -112,6 +119,8 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
       << Timestamp::kTimestampUnitsPerSecond;
 
   frame_time_usec_ = static_cast<int64>(1000000.0 / frame_rate_);
+  jitter_usec_ = static_cast<int64>(1000000.0 * jitter_ / frame_rate_);
+  RET_CHECK_LE(jitter_usec_, frame_time_usec_);
 
   video_header_.frame_rate = frame_rate_;
 
@@ -188,12 +197,32 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
 
 void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
   next_output_timestamp_min_ = first_timestamp_;
+  if (jitter_with_reflection_) {
+    next_output_timestamp_ =
+        first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
+    return;
+  }
   next_output_timestamp_ =
       first_timestamp_ + frame_time_usec_ * random_->RandFloat();
 }
 
 void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
   packet_reservoir_->Clear();
+  if (jitter_with_reflection_) {
+    next_output_timestamp_min_ += frame_time_usec_;
+    Timestamp next_output_timestamp_max_ =
+        next_output_timestamp_min_ + frame_time_usec_;
+
+    next_output_timestamp_ += frame_time_usec_ +
+                              random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
+                              jitter_usec_;
+    next_output_timestamp_ = Timestamp(ReflectBetween(
+        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+        next_output_timestamp_max_.Value()));
+    CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
+    CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
+    return;
+  }
   packet_reservoir_->Disable();
   next_output_timestamp_ +=
       frame_time_usec_ *

mediapipe/calculators/core/packet_resampler_calculator.h

@@ -49,6 +49,38 @@ class PacketReservoir {
 // out of a stream. Given a desired frame rate, packets are going to be
 // removed or added to achieve it.
 //
+// If jitter_ is specified:
+//   - The first packet is chosen randomly (uniform distribution) among frames
+//     that correspond to timestamps [0, 1/frame_rate).  Let the chosen packet
+//     correspond to timestamp t.
+//   - The next packet is chosen randomly (uniform distribution) among frames
+//     that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
+//     - if jitter_with_reflection_ is true, the timestamp will be reflected
+//       against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
+//       so that its marginal distribution is uniform within this interval.
+//       In the formula, t_0 is the timestamp of the first sampled
+//       packet, and the k is the packet index.
+//       See paper (https://arxiv.org/abs/2002.01147) for details.
+//   - t is updated and the process is repeated.
+//   - Note that seed is specified as input side packet for reproducibility of
+//     the resampling.  For Cloud ML Video Intelligence API, the hash of the
+//     input video should serve this purpose.  For YouTube, either video ID or
+//     content hex ID of the input video should do.
+//
+// If jitter_ is not specified:
+//   - The first packet defines the first_timestamp of the output stream,
+//     so it is always emitted.
+//   - If more packets are emitted, they will have timestamp equal to
+//     round(first_timestamp + k * period) , where k is a positive
+//     integer and the period is defined by the frame rate.
+//     Example: first_timestamp=0, fps=30, then the output stream
+//              will have timestamps: 0, 33333, 66667, 100000, etc...
+//   - The packets selected for the output stream are the ones closer
+//     to the exact middle point (33333.33, 66666.67 in our previous
+//     example). In case of ties, later packets are chosen.
+//   - 'Empty' periods happen when there are no packets for a long time
+//     (greater than a period). In this case, we send a copy of the last
+//     packet received before the empty period.
 // The jitter feature is disabled by default. To enable it, you need to
 // implement CreateSecureRandom(const std::string&).
 //
@@ -139,7 +171,12 @@ class PacketResamplerCalculator : public CalculatorBase {
   // Jitter-related variables.
   std::unique_ptr<RandomBase> random_;
   double jitter_ = 0.0;
+  bool jitter_with_reflection_;
+  int64 jitter_usec_;
   Timestamp next_output_timestamp_;
+  // If jittering_with_reflection_ is true, next_output_timestamp_ will be
+  // kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
   Timestamp next_output_timestamp_min_;
 
   // If specified, output timestamps are aligned with base_timestamp.

mediapipe/calculators/core/packet_resampler_calculator.proto

@@ -66,6 +66,7 @@ message PacketResamplerCalculatorOptions {
   // pseudo-random number generator does its job and the number of frames is
   // sufficiently large, the average frame rate will be close to this value.
   optional double jitter = 4;
+  optional bool jitter_with_reflection = 9 [default = false];
 
   // If specified, output timestamps are aligned with base_timestamp.
   // Otherwise, they are aligned with the first input timestamp.

mediapipe/calculators/image/BUILD

@@ -332,6 +332,7 @@ cc_library(
 cc_library(
     name = "image_cropping_calculator",
     srcs = ["image_cropping_calculator.cc"],
+    hdrs = ["image_cropping_calculator.h"],
     copts = select({
         "//mediapipe:apple": [
             "-x objective-c++",
@@ -371,6 +372,22 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_test(
+    name = "image_cropping_calculator_test",
+    srcs = ["image_cropping_calculator_test.cc"],
+    deps = [
+        ":image_cropping_calculator",
+        ":image_cropping_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:tag_map",
+        "//mediapipe/framework/tool:tag_map_helper",
+    ],
+)
+
 cc_library(
     name = "luminance_calculator",
     srcs = ["luminance_calculator.cc"],

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/image/image_cropping_calculator.h"
+
 #include <cmath>
 
-#include "mediapipe/calculators/image/image_cropping_calculator.pb.h"
-#include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/rect.pb.h"
@@ -25,7 +25,6 @@
 #include "mediapipe/framework/port/status.h"
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-#include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gl_simple_shaders.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/gpu/shader_util.h"
@@ -52,62 +51,6 @@ constexpr char kWidthTag[] = "WIDTH";
 
 }  // namespace
 
-// Crops the input texture to the given rectangle region. The rectangle can
-// be at arbitrary location on the image with rotation. If there's rotation, the
-// output texture will have the size of the input rectangle. The rotation should
-// be in radian, see rect.proto for detail.
-//
-// Input:
-//   One of the following two tags:
-//   IMAGE - ImageFrame representing the input image.
-//   IMAGE_GPU - GpuBuffer representing the input image.
-//   One of the following two tags (optional if WIDTH/HEIGHT is specified):
-//   RECT - A Rect proto specifying the width/height and location of the
-//          cropping rectangle.
-//   NORM_RECT - A NormalizedRect proto specifying the width/height and location
-//               of the cropping rectangle in normalized coordinates.
-//   Alternative tags to RECT (optional if RECT/NORM_RECT is specified):
-//   WIDTH - The desired width of the output cropped image,
-//           based on image center
-//   HEIGHT - The desired height of the output cropped image,
-//            based on image center
-//
-// Output:
-//   One of the following two tags:
-//   IMAGE - Cropped ImageFrame
-//   IMAGE_GPU - Cropped GpuBuffer.
-//
-// Note: input_stream values take precedence over options defined in the graph.
-//
-class ImageCroppingCalculator : public CalculatorBase {
- public:
-  ImageCroppingCalculator() = default;
-  ~ImageCroppingCalculator() override = default;
-
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-  ::mediapipe::Status Close(CalculatorContext* cc) override;
-
- private:
-  ::mediapipe::Status RenderCpu(CalculatorContext* cc);
-  ::mediapipe::Status RenderGpu(CalculatorContext* cc);
-  ::mediapipe::Status InitGpu(CalculatorContext* cc);
-  void GlRender();
-  void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
-                           int* dst_width, int* dst_height);
-
-  mediapipe::ImageCroppingCalculatorOptions options_;
-
-  bool use_gpu_ = false;
-  // Output texture corners (4) after transoformation in normalized coordinates.
-  float transformed_points_[8];
-#if !defined(MEDIAPIPE_DISABLE_GPU)
-  bool gpu_initialized_ = false;
-  mediapipe::GlCalculatorHelper gpu_helper_;
-  GLuint program_ = 0;
-#endif  //  !MEDIAPIPE_DISABLE_GPU
-};
 REGISTER_CALCULATOR(ImageCroppingCalculator);
 
 ::mediapipe::Status ImageCroppingCalculator::GetContract(
@@ -132,7 +75,11 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-  RET_CHECK(cc->Inputs().HasTag(kRectTag) ^ cc->Inputs().HasTag(kNormRectTag));
+  RET_CHECK(cc->Inputs().HasTag(kRectTag) ^ cc->Inputs().HasTag(kNormRectTag) ^
+            (cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
+                 .has_norm_width() &&
+             cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
+                 .has_norm_height()));
   if (cc->Inputs().HasTag(kRectTag)) {
     cc->Inputs().Tag(kRectTag).Set<Rect>();
   }
@@ -222,41 +169,8 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   const auto& input_img = cc->Inputs().Tag(kImageTag).Get<ImageFrame>();
   cv::Mat input_mat = formats::MatView(&input_img);
 
-  float rect_center_x = input_img.Width() / 2.0f;
-  float rect_center_y = input_img.Height() / 2.0f;
-  float rotation = 0.0f;
-  int target_width = input_img.Width();
-  int target_height = input_img.Height();
-  if (cc->Inputs().HasTag(kRectTag)) {
-    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
-    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
-        rect.y_center() >= 0) {
-      rect_center_x = rect.x_center();
-      rect_center_y = rect.y_center();
-      target_width = rect.width();
-      target_height = rect.height();
-      rotation = rect.rotation();
-    }
-  } else if (cc->Inputs().HasTag(kNormRectTag)) {
-    const auto& rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
-    if (rect.width() > 0.0 && rect.height() > 0.0 && rect.x_center() >= 0.0 &&
-        rect.y_center() >= 0.0) {
-      rect_center_x = std::round(rect.x_center() * input_img.Width());
-      rect_center_y = std::round(rect.y_center() * input_img.Height());
-      target_width = std::round(rect.width() * input_img.Width());
-      target_height = std::round(rect.height() * input_img.Height());
-      rotation = rect.rotation();
-    }
-  } else {
-    if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
-      target_width = cc->Inputs().Tag(kWidthTag).Get<int>();
-      target_height = cc->Inputs().Tag(kHeightTag).Get<int>();
-    } else if (options_.has_width() && options_.has_height()) {
-      target_width = options_.width();
-      target_height = options_.height();
-    }
-    rotation = options_.rotation();
-  }
+  auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
+      GetCropSpecs(cc, input_img.Width(), input_img.Height());
 
   const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y),
                                  cv::Size2f(target_width, target_height),
@@ -433,46 +347,8 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
                                                   int src_width, int src_height,
                                                   int* dst_width,
                                                   int* dst_height) {
-  // Get the size of the cropping box.
-  int crop_width = src_width;
-  int crop_height = src_height;
-  // Get the center of cropping box. Default is the at the center.
-  int x_center = src_width / 2;
-  int y_center = src_height / 2;
-  // Get the rotation of the cropping box.
-  float rotation = 0.0f;
-  if (cc->Inputs().HasTag(kRectTag)) {
-    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
-    // Only use the rect if it is valid.
-    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
-        rect.y_center() >= 0) {
-      x_center = rect.x_center();
-      y_center = rect.y_center();
-      crop_width = rect.width();
-      crop_height = rect.height();
-      rotation = rect.rotation();
-    }
-  } else if (cc->Inputs().HasTag(kNormRectTag)) {
-    const auto& rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
-    // Only use the rect if it is valid.
-    if (rect.width() > 0.0 && rect.height() > 0.0 && rect.x_center() >= 0.0 &&
-        rect.y_center() >= 0.0) {
-      x_center = std::round(rect.x_center() * src_width);
-      y_center = std::round(rect.y_center() * src_height);
-      crop_width = std::round(rect.width() * src_width);
-      crop_height = std::round(rect.height() * src_height);
-      rotation = rect.rotation();
-    }
-  } else {
-    if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
-      crop_width = cc->Inputs().Tag(kWidthTag).Get<int>();
-      crop_height = cc->Inputs().Tag(kHeightTag).Get<int>();
-    } else if (options_.has_width() && options_.has_height()) {
-      crop_width = options_.width();
-      crop_height = options_.height();
-    }
-    rotation = options_.rotation();
-  }
+  auto [crop_width, crop_height, x_center, y_center, rotation] =
+      GetCropSpecs(cc, src_width, src_height);
 
   const float half_width = crop_width / 2.0f;
   const float half_height = crop_height / 2.0f;
@@ -508,4 +384,82 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
   *dst_height = std::max(1, height);
 }
 
+RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
+                                               int src_width, int src_height) {
+  // Get the size of the cropping box.
+  int crop_width = src_width;
+  int crop_height = src_height;
+  // Get the center of cropping box. Default is the at the center.
+  int x_center = src_width / 2;
+  int y_center = src_height / 2;
+  // Get the rotation of the cropping box.
+  float rotation = 0.0f;
+  // Get the normalized width and height if specified by the inputs or options.
+  float normalized_width = 0.0f;
+  float normalized_height = 0.0f;
+
+  mediapipe::ImageCroppingCalculatorOptions options =
+      cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
+
+  // width/height, norm_width/norm_height from input streams take precednece.
+  if (cc->Inputs().HasTag(kRectTag)) {
+    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
+    // Only use the rect if it is valid.
+    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
+        rect.y_center() >= 0) {
+      x_center = rect.x_center();
+      y_center = rect.y_center();
+      crop_width = rect.width();
+      crop_height = rect.height();
+      rotation = rect.rotation();
+    }
+  } else if (cc->Inputs().HasTag(kNormRectTag)) {
+    const auto& norm_rect =
+        cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
+    if (norm_rect.width() > 0.0 && norm_rect.height() > 0.0) {
+      normalized_width = norm_rect.width();
+      normalized_height = norm_rect.height();
+      x_center = std::round(norm_rect.x_center() * src_width);
+      y_center = std::round(norm_rect.y_center() * src_height);
+      rotation = norm_rect.rotation();
+    }
+  } else if (cc->Inputs().HasTag(kWidthTag) &&
+             cc->Inputs().HasTag(kHeightTag)) {
+    crop_width = cc->Inputs().Tag(kWidthTag).Get<int>();
+    crop_height = cc->Inputs().Tag(kHeightTag).Get<int>();
+  } else if (options.has_width() && options.has_height()) {
+    crop_width = options.width();
+    crop_height = options.height();
+  } else if (options.has_norm_width() && options.has_norm_height()) {
+    normalized_width = options.norm_width();
+    normalized_height = options.norm_height();
+  }
+
+  // Get the crop width and height from the normalized width and height.
+  if (normalized_width > 0 && normalized_height > 0) {
+    crop_width = std::round(normalized_width * src_width);
+    crop_height = std::round(normalized_height * src_height);
+  }
+
+  // Rotation and center values from input streams take precedence, so only
+  // look at those values in the options if kRectTag and kNormRectTag are not
+  // present from the inputs.
+  if (!cc->Inputs().HasTag(kRectTag) && !cc->Inputs().HasTag(kNormRectTag)) {
+    if (options.has_norm_center_x() && options.has_norm_center_y()) {
+      x_center = std::round(options.norm_center_x() * src_width);
+      y_center = std::round(options.norm_center_y() * src_height);
+    }
+    if (options.has_rotation()) {
+      rotation = options.rotation();
+    }
+  }
+  return {
+      .width = crop_width,
+      .height = crop_height,
+      .center_x = x_center,
+      .center_y = y_center,
+      .rotation = rotation,
+  };
+}
+
 }  // namespace mediapipe

mediapipe/calculators/image/image_cropping_calculator.h

@@ -0,0 +1,86 @@
+#ifndef MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_
+
+#include "mediapipe/calculators/image/image_cropping_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+// Crops the input texture to the given rectangle region. The rectangle can
+// be at arbitrary location on the image with rotation. If there's rotation, the
+// output texture will have the size of the input rectangle. The rotation should
+// be in radian, see rect.proto for detail.
+//
+// Input:
+//   One of the following two tags:
+//   IMAGE - ImageFrame representing the input image.
+//   IMAGE_GPU - GpuBuffer representing the input image.
+//   One of the following two tags (optional if WIDTH/HEIGHT is specified):
+//   RECT - A Rect proto specifying the width/height and location of the
+//          cropping rectangle.
+//   NORM_RECT - A NormalizedRect proto specifying the width/height and location
+//               of the cropping rectangle in normalized coordinates.
+//   Alternative tags to RECT (optional if RECT/NORM_RECT is specified):
+//   WIDTH - The desired width of the output cropped image,
+//           based on image center
+//   HEIGHT - The desired height of the output cropped image,
+//            based on image center
+//
+// Output:
+//   One of the following two tags:
+//   IMAGE - Cropped ImageFrame
+//   IMAGE_GPU - Cropped GpuBuffer.
+//
+// Note: input_stream values take precedence over options defined in the graph.
+//
+namespace mediapipe {
+struct RectSpec {
+  int width;
+  int height;
+  int center_x;
+  int center_y;
+  float rotation;
+
+  bool operator==(const RectSpec& rect) const {
+    return (width == rect.width && height == rect.height &&
+            center_x == rect.center_x && center_y == rect.center_y &&
+            rotation == rect.rotation);
+  }
+};
+
+class ImageCroppingCalculator : public CalculatorBase {
+ public:
+  ImageCroppingCalculator() = default;
+  ~ImageCroppingCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+  static RectSpec GetCropSpecs(const CalculatorContext* cc, int src_width,
+                               int src_height);
+
+ private:
+  ::mediapipe::Status RenderCpu(CalculatorContext* cc);
+  ::mediapipe::Status RenderGpu(CalculatorContext* cc);
+  ::mediapipe::Status InitGpu(CalculatorContext* cc);
+  void GlRender();
+  void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
+                           int* dst_width, int* dst_height);
+
+  mediapipe::ImageCroppingCalculatorOptions options_;
+
+  bool use_gpu_ = false;
+  // Output texture corners (4) after transoformation in normalized coordinates.
+  float transformed_points_[8];
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  bool gpu_initialized_ = false;
+  mediapipe::GlCalculatorHelper gpu_helper_;
+  GLuint program_ = 0;
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+};
+
+}  // namespace mediapipe
+#endif  // MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_

mediapipe/calculators/image/image_cropping_calculator.proto

@@ -30,4 +30,14 @@ message ImageCroppingCalculatorOptions {
 
   // Rotation angle is counter-clockwise in radian.
   optional float rotation = 3 [default = 0.0];
+
+  // Normalized width and height of the output rect. Value is within [0, 1].
+  optional float norm_width = 4;
+  optional float norm_height = 5;
+
+  // Normalized location of the center of the output
+  // rectangle in image coordinates. Value is within [0, 1].
+  // The (0, 0) point is at the (top, left) corner.
+  optional float norm_center_x = 6 [default = 0];
+  optional float norm_center_y = 7 [default = 0];
 }

mediapipe/calculators/image/image_cropping_calculator_test.cc

@@ -0,0 +1,216 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/image/image_cropping_calculator.h"
+
+#include <cmath>
+#include <memory>
+
+#include "mediapipe/calculators/image/image_cropping_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/tag_map.h"
+#include "mediapipe/framework/tool/tag_map_helper.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr int input_width = 100;
+constexpr int input_height = 100;
+
+constexpr char kRectTag[] = "RECT";
+constexpr char kHeightTag[] = "HEIGHT";
+constexpr char kWidthTag[] = "WIDTH";
+
+// Test normal case, where norm_width and norm_height in options are set.
+TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
+  auto calculator_node =
+      ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
+          R"(
+            calculator: "ImageCroppingCalculator"
+            input_stream: "IMAGE_GPU:input_frames"
+            output_stream: "IMAGE_GPU:cropped_output_frames"
+            options: {
+              [mediapipe.ImageCroppingCalculatorOptions.ext] {
+                norm_width: 0.6
+                norm_height: 0.6
+                norm_center_x: 0.5
+                norm_center_y: 0.5
+                rotation: 0.3
+              }
+            }
+          )");
+
+  auto calculator_state =
+      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+  auto cc =
+      CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
+                        tool::CreateTagMap({}).ValueOrDie());
+
+  RectSpec expectRect = {
+      .width = 60,
+      .height = 60,
+      .center_x = 50,
+      .center_y = 50,
+      .rotation = 0.3,
+  };
+  EXPECT_EQ(
+      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+      expectRect);
+}  // TEST
+
+// Test when (width height) + (norm_width norm_height) are set in options.
+// width and height should take precedence.
+TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
+  auto calculator_node =
+      ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
+          R"(
+            calculator: "ImageCroppingCalculator"
+            input_stream: "IMAGE_GPU:input_frames"
+            output_stream: "IMAGE_GPU:cropped_output_frames"
+            options: {
+              [mediapipe.ImageCroppingCalculatorOptions.ext] {
+                width: 50
+                height: 50
+                norm_width: 0.6
+                norm_height: 0.6
+                norm_center_x: 0.5
+                norm_center_y: 0.5
+                rotation: 0.3
+              }
+            }
+          )");
+
+  auto calculator_state =
+      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+  auto cc =
+      CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
+                        tool::CreateTagMap({}).ValueOrDie());
+  RectSpec expectRect = {
+      .width = 50,
+      .height = 50,
+      .center_x = 50,
+      .center_y = 50,
+      .rotation = 0.3,
+  };
+  EXPECT_EQ(
+      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+      expectRect);
+}  // TEST
+
+// Test when WIDTH HEIGHT are set from input stream,
+// and options has norm_width/height set.
+// WIDTH HEIGHT from input stream should take precedence.
+TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
+  auto calculator_node =
+      ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
+          R"(
+            calculator: "ImageCroppingCalculator"
+            input_stream: "IMAGE_GPU:input_frames"
+            input_stream: "WIDTH:crop_width"
+            input_stream: "HEIGHT:crop_height"
+            output_stream: "IMAGE_GPU:cropped_output_frames"
+            options: {
+              [mediapipe.ImageCroppingCalculatorOptions.ext] {
+                width: 50
+                height: 50
+                norm_width: 0.6
+                norm_height: 0.6
+                norm_center_x: 0.5
+                norm_center_y: 0.5
+                rotation: 0.3
+              }
+            }
+          )");
+
+  auto calculator_state =
+      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+  auto inputTags = tool::CreateTagMap({
+                                          "HEIGHT:0:crop_height",
+                                          "WIDTH:0:crop_width",
+                                      })
+                       .ValueOrDie();
+  auto cc = CalculatorContext(&calculator_state, inputTags,
+                              tool::CreateTagMap({}).ValueOrDie());
+  auto& inputs = cc.Inputs();
+  inputs.Tag(kHeightTag).Value() = MakePacket<int>(1);
+  inputs.Tag(kWidthTag).Value() = MakePacket<int>(1);
+  RectSpec expectRect = {
+      .width = 1,
+      .height = 1,
+      .center_x = 50,
+      .center_y = 50,
+      .rotation = 0.3,
+  };
+  EXPECT_EQ(
+      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+      expectRect);
+}  // TEST
+
+// Test when RECT is set from input stream,
+// and options has norm_width/height set.
+// RECT from input stream should take precedence.
+TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
+  auto calculator_node =
+      ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
+          R"(
+            calculator: "ImageCroppingCalculator"
+            input_stream: "IMAGE_GPU:input_frames"
+            input_stream: "RECT:rect"
+            output_stream: "IMAGE_GPU:cropped_output_frames"
+            options: {
+              [mediapipe.ImageCroppingCalculatorOptions.ext] {
+                width: 50
+                height: 50
+                norm_width: 0.6
+                norm_height: 0.6
+                norm_center_x: 0.5
+                norm_center_y: 0.5
+                rotation: 0.3
+              }
+            }
+          )");
+
+  auto calculator_state =
+      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+  auto inputTags = tool::CreateTagMap({
+                                          "RECT:0:rect",
+                                      })
+                       .ValueOrDie();
+  auto cc = CalculatorContext(&calculator_state, inputTags,
+                              tool::CreateTagMap({}).ValueOrDie());
+  auto& inputs = cc.Inputs();
+  mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
+      R"(
+        width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
+      )");
+  inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect);
+  RectSpec expectRect = {
+      .width = 1,
+      .height = 1,
+      .center_x = 40,
+      .center_y = 40,
+      .rotation = 0.5,
+  };
+  EXPECT_EQ(
+      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+      expectRect);
+}  // TEST
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -138,8 +138,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 // Note: To enable horizontal or vertical flipping, specify them in the
 // calculator options. Flipping is applied after rotation.
 //
-// Note: Only scale mode STRETCH is currently supported on CPU,
-// and flipping is not yet supported either.
+// Note: Only scale mode STRETCH is currently supported on CPU.
 //
 class ImageTransformationCalculator : public CalculatorBase {
  public:
@@ -316,6 +315,11 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   cv::Mat input_mat = formats::MatView(&input_img);
   cv::Mat scaled_mat;
 
+  if (!output_height_ || !output_width_) {
+    output_height_ = input_height;
+    output_width_ = input_width;
+  }
+
   if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
     cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_));
   } else {
@@ -367,10 +371,21 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   cv::Mat rotation_mat = cv::getRotationMatrix2D(src_center, angle, 1.0);
   cv::warpAffine(scaled_mat, rotated_mat, rotation_mat, scaled_mat.size());
 
+  cv::Mat flipped_mat;
+  if (options_.flip_horizontally() || options_.flip_vertically()) {
+    const int flip_code =
+        options_.flip_horizontally() && options_.flip_vertically()
+            ? -1
+            : options_.flip_horizontally();
+    cv::flip(rotated_mat, flipped_mat, flip_code);
+  } else {
+    flipped_mat = rotated_mat;
+  }
+
   std::unique_ptr<ImageFrame> output_frame(
       new ImageFrame(input_img.Format(), output_width, output_height));
   cv::Mat output_mat = formats::MatView(output_frame.get());
-  rotated_mat.copyTo(output_mat);
+  flipped_mat.copyTo(output_mat);
   cc->Outputs().Tag("IMAGE").Add(output_frame.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
@@ -440,9 +455,8 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
         cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
   }
 
-  static mediapipe::FrameScaleMode scale_mode =
-      mediapipe::FrameScaleModeFromProto(scale_mode_,
-                                         mediapipe::FrameScaleMode::kStretch);
+  mediapipe::FrameScaleMode scale_mode = mediapipe::FrameScaleModeFromProto(
+      scale_mode_, mediapipe::FrameScaleMode::kStretch);
   mediapipe::FrameRotation rotation =
       mediapipe::FrameRotationFromDegrees(RotationModeToDegrees(rotation_));
 

mediapipe/calculators/image/testdata/BUILD

@@ -21,6 +21,7 @@ filegroup(
         "dino.jpg",
         "dino_quality_50.jpg",
         "dino_quality_80.jpg",
+        "front_camera_pixel2.jpg",
     ],
     visibility = ["//visibility:public"],
 )

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -34,6 +34,7 @@ namespace mediapipe {
 
 const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
 const char kImageTag[] = "IMAGE";
+const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_";
 const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_";
 const char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
 const char kBBoxTag[] = "BBOX";
@@ -145,6 +146,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
         }
         cc->Inputs().Tag(tag).Set<std::vector<Detection>>();
       }
+      if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag)) {
+        cc->Inputs().Tag(tag).Set<std::vector<float>>();
+      }
       if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
         cc->Inputs().Tag(tag).Set<std::vector<float>>();
       }
@@ -344,6 +348,17 @@ class PackMediaSequenceCalculator : public CalculatorBase {
                              sequence_.get());
         }
       }
+      if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag) &&
+          !cc->Inputs().Tag(tag).IsEmpty()) {
+        std::string key =
+            tag.substr(sizeof(kFloatContextFeaturePrefixTag) /
+                           sizeof(*kFloatContextFeaturePrefixTag) -
+                       1);
+        RET_CHECK_EQ(cc->InputTimestamp(), Timestamp::PostStream());
+        mpms::SetContextFeatureFloats(
+            key, cc->Inputs().Tag(tag).Get<std::vector<float>>(),
+            sequence_.get());
+      }
       if (absl::StartsWith(tag, kFloatFeaturePrefixTag) &&
           !cc->Inputs().Tag(tag).IsEmpty()) {
         std::string key = tag.substr(sizeof(kFloatFeaturePrefixTag) /

mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc

@@ -194,6 +194,38 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) {
   }
 }
 
+TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) {
+  SetUpCalculator(
+      {"FLOAT_CONTEXT_FEATURE_TEST:test", "FLOAT_CONTEXT_FEATURE_OTHER:test2"},
+      {}, false, true);
+  auto input_sequence = absl::make_unique<tf::SequenceExample>();
+
+  auto vf_ptr = absl::make_unique<std::vector<float>>(2, 3);
+  runner_->MutableInputs()
+      ->Tag("FLOAT_CONTEXT_FEATURE_TEST")
+      .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
+  vf_ptr = absl::make_unique<std::vector<float>>(2, 4);
+  runner_->MutableInputs()
+      ->Tag("FLOAT_CONTEXT_FEATURE_OTHER")
+      .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
+
+  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+      Adopt(input_sequence.release());
+
+  MP_ASSERT_OK(runner_->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const tf::SequenceExample& output_sequence =
+      output_packets[0].Get<tf::SequenceExample>();
+
+  ASSERT_THAT(mpms::GetContextFeatureFloats("TEST", output_sequence),
+              testing::ElementsAre(3, 3));
+  ASSERT_THAT(mpms::GetContextFeatureFloats("OTHER", output_sequence),
+              testing::ElementsAre(4, 4));
+}
+
 TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
   tf::Features context;
   (*context.mutable_feature())["TEST"].mutable_bytes_list()->add_value("YES");

mediapipe/calculators/util/BUILD

@@ -508,6 +508,17 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "timed_box_list_to_render_data_calculator_proto",
+    srcs = ["timed_box_list_to_render_data_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util:color_proto",
+        "//mediapipe/util:render_data_proto",
+    ],
+)
+
 proto_library(
     name = "labels_to_render_data_calculator_proto",
     srcs = ["labels_to_render_data_calculator.proto"],
@@ -651,6 +662,37 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_cc_proto_library(
+    name = "timed_box_list_to_render_data_calculator_cc_proto",
+    srcs = ["timed_box_list_to_render_data_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util:color_cc_proto",
+        "//mediapipe/util:render_data_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":timed_box_list_to_render_data_calculator_proto"],
+)
+
+cc_library(
+    name = "timed_box_list_to_render_data_calculator",
+    srcs = ["timed_box_list_to_render_data_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":timed_box_list_to_render_data_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_options_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/util:color_cc_proto",
+        "//mediapipe/util:render_data_cc_proto",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "//mediapipe/util/tracking:tracking_cc_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "labels_to_render_data_calculator",
     srcs = ["labels_to_render_data_calculator.cc"],

mediapipe/calculators/util/collection_has_min_size_calculator.h

@@ -37,6 +37,8 @@ namespace mediapipe {
 //    }
 //   }
 // }
+// Optionally, uses a side packet to override `min_size` specified in the
+// calculator options.
 template <typename IterableT>
 class CollectionHasMinSizeCalculator : public CalculatorBase {
  public:
@@ -54,6 +56,10 @@ class CollectionHasMinSizeCalculator : public CalculatorBase {
     cc->Inputs().Tag("ITERABLE").Set<IterableT>();
     cc->Outputs().Index(0).Set<bool>();
 
+    // Optional input side packet that determines `min_size_`.
+    if (cc->InputSidePackets().NumEntries() > 0) {
+      cc->InputSidePackets().Index(0).Set<int>();
+    }
     return ::mediapipe::OkStatus();
   }
 
@@ -62,6 +68,11 @@ class CollectionHasMinSizeCalculator : public CalculatorBase {
     min_size_ =
         cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>()
             .min_size();
+    // Override `min_size` if passed as side packet.
+    if (cc->InputSidePackets().NumEntries() > 0 &&
+        !cc->InputSidePackets().Index(0).IsEmpty()) {
+      min_size_ = cc->InputSidePackets().Index(0).Get<int>();
+    }
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc

@@ -0,0 +1,147 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/memory/memory.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
+#include "mediapipe/calculators/util/timed_box_list_to_render_data_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_options.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/color.pb.h"
+#include "mediapipe/util/render_data.pb.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+#include "mediapipe/util/tracking/tracking.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kTimedBoxListTag[] = "BOX_LIST";
+constexpr char kRenderDataTag[] = "RENDER_DATA";
+
+void AddTimedBoxProtoToRenderData(
+    const TimedBoxProto& box_proto,
+    const TimedBoxListToRenderDataCalculatorOptions& options,
+    RenderData* render_data) {
+  if (box_proto.has_quad() && box_proto.quad().vertices_size() > 0 &&
+      box_proto.quad().vertices_size() % 2 == 0) {
+    const int num_corners = box_proto.quad().vertices_size() / 2;
+    for (int i = 0; i < num_corners; ++i) {
+      const int next_corner = (i + 1) % num_corners;
+      auto* line_annotation = render_data->add_render_annotations();
+      line_annotation->mutable_color()->set_r(options.box_color().r());
+      line_annotation->mutable_color()->set_g(options.box_color().g());
+      line_annotation->mutable_color()->set_b(options.box_color().b());
+      line_annotation->set_thickness(options.thickness());
+      RenderAnnotation::Line* line = line_annotation->mutable_line();
+      line->set_normalized(true);
+      line->set_x_start(box_proto.quad().vertices(i * 2));
+      line->set_y_start(box_proto.quad().vertices(i * 2 + 1));
+      line->set_x_end(box_proto.quad().vertices(next_corner * 2));
+      line->set_y_end(box_proto.quad().vertices(next_corner * 2 + 1));
+    }
+  } else {
+    auto* rect_annotation = render_data->add_render_annotations();
+    rect_annotation->mutable_color()->set_r(options.box_color().r());
+    rect_annotation->mutable_color()->set_g(options.box_color().g());
+    rect_annotation->mutable_color()->set_b(options.box_color().b());
+    rect_annotation->set_thickness(options.thickness());
+    RenderAnnotation::Rectangle* rect = rect_annotation->mutable_rectangle();
+    rect->set_normalized(true);
+    rect->set_left(box_proto.left());
+    rect->set_right(box_proto.right());
+    rect->set_top(box_proto.top());
+    rect->set_bottom(box_proto.bottom());
+    rect->set_rotation(box_proto.rotation());
+  }
+}
+
+}  // namespace
+
+// A calculator that converts TimedBoxProtoList proto to RenderData proto for
+// visualization. If the input TimedBoxProto contains `quad` field, this
+// calculator will draw a quadrilateral based on it. Otherwise this calculator
+// will draw a rotated rectangle based on `top`, `bottom`, `left`, `right` and
+// `rotation` fields
+//
+// Example config:
+// node {
+//   calculator: "TimedBoxListToRenderDataCalculator"
+//   input_stream: "BOX_LIST:landmarks"
+//   output_stream: "RENDER_DATA:render_data"
+//   options {
+//     [TimedBoxListToRenderDataCalculatorOptions.ext] {
+//       box_color { r: 0 g: 255 b: 0 }
+//       thickness: 4.0
+//     }
+//   }
+// }
+class TimedBoxListToRenderDataCalculator : public CalculatorBase {
+ public:
+  TimedBoxListToRenderDataCalculator() {}
+  ~TimedBoxListToRenderDataCalculator() override {}
+  TimedBoxListToRenderDataCalculator(
+      const TimedBoxListToRenderDataCalculator&) = delete;
+  TimedBoxListToRenderDataCalculator& operator=(
+      const TimedBoxListToRenderDataCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  TimedBoxListToRenderDataCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(TimedBoxListToRenderDataCalculator);
+
+::mediapipe::Status TimedBoxListToRenderDataCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().HasTag(kTimedBoxListTag)) {
+    cc->Inputs().Tag(kTimedBoxListTag).Set<TimedBoxProtoList>();
+  }
+  cc->Outputs().Tag(kRenderDataTag).Set<RenderData>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TimedBoxListToRenderDataCalculator::Open(
+    CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+  options_ = cc->Options<TimedBoxListToRenderDataCalculatorOptions>();
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TimedBoxListToRenderDataCalculator::Process(
+    CalculatorContext* cc) {
+  auto render_data = absl::make_unique<RenderData>();
+
+  if (cc->Inputs().HasTag(kTimedBoxListTag)) {
+    const auto& box_list =
+        cc->Inputs().Tag(kTimedBoxListTag).Get<TimedBoxProtoList>();
+
+    for (const auto& box : box_list.box()) {
+      AddTimedBoxProtoToRenderData(box, options_, render_data.get());
+    }
+  }
+
+  cc->Outputs()
+      .Tag(kRenderDataTag)
+      .Add(render_data.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/timed_box_list_to_render_data_calculator.proto

@@ -0,0 +1,32 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/color.proto";
+
+message TimedBoxListToRenderDataCalculatorOptions {
+  extend CalculatorOptions {
+    optional TimedBoxListToRenderDataCalculatorOptions ext = 289899854;
+  }
+
+  // Color of boxes.
+  optional Color box_color = 1;
+
+  // Thickness of the drawing of boxes.
+  optional double thickness = 2 [default = 1.0];
+}

mediapipe/calculators/video/tvl1_optical_flow_calculator.cc

@@ -88,7 +88,8 @@ class Tvl1OpticalFlowCalculator : public CalculatorBase {
   // cv::DenseOpticalFlow is not thread-safe. Invoking multiple
   // DenseOpticalFlow::calc() in parallel may lead to memory corruption or
   // memory leak.
-  std::list<cv::Ptr<cv::DenseOpticalFlow>> tvl1_computers_ GUARDED_BY(mutex_);
+  std::list<cv::Ptr<cv::DenseOpticalFlow>> tvl1_computers_
+      ABSL_GUARDED_BY(mutex_);
   absl::Mutex mutex_;
 };
 

mediapipe/docs/autoflip.md

@@ -24,13 +24,11 @@ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/a
 ```bash
 GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/autoflip/run_autoflip \
   --calculator_graph_config_file=mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt \
-  --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,\
-    output_video_path=/absolute/path/to/save/the/output/video/file,\
-    aspect_ratio=9:16
+  --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,output_video_path=/absolute/path/to/save/the/output/video/file,aspect_ratio=1:1
 ```
 
 Use the `aspect_ratio` flag to provide the output aspect ratio. The format
-should be `weight:height`, where the `weight` and `height` are two positive
+should be `width:height`, where the `width` and `height` are two positive
 integers. AutoFlip supports both landscape-to-portrait and portrait-to-landscape
 conversions. The pipeline internally compares the target aspect ratio against
 the original one, and determines the correct conversion automatically.
@@ -85,7 +83,7 @@ node {
   node_options: {
     [type.googleapis.com/mediapipe.PacketThinnerCalculatorOptions]: {
       thinner_type: ASYNC
-      period: 500000
+      period: 200000
     }
   }
 }
@@ -215,7 +213,7 @@ node {
         score_aggregation_type: CONSTANT
       }
       scene_camera_motion_analyzer_options: {
-        motion_stabilization_threshold_percent: 0.3
+        motion_stabilization_threshold_percent: 0.5
         salient_point_bound: 0.499
       }
       padding_parameters: {
@@ -280,6 +278,28 @@ salient content as possible. See an illustration below.
 
 ![graph is_required](images/autoflip_is_required.gif)
 
+### Stable vs Tracking Camera Motion
+
+AutoFlip makes a decision on each scene whether to have the cropped viewpoint
+follow an object or if the crop should remain stable (centered on detected
+objects). The parameter `motion_stabilization_threshold_percent` value is used
+to make the decision to track action or keep the camera stable. If, over the
+duration of the scene, all detected focus objects remain within this ratio of
+the frame (e.g. 0.5 = 50% or 1920 * .5 = 960 pixels on 1080p video) then the
+camera is held steady. Otherwise the camera tracks activity within the frame.
+
+### Snap To Center
+
+For some scenes the camera viewpoint will remain stable at the center of
+activity (see `motion_stabilization_threshold_percent` setting). In this case,
+if the determined best stable viewpoint is within
+`snap_center_max_distance_percent` of the frame's center the camera will be
+shifted to be locked to the center of the frame. This setting is useful for
+videos where the camera operator did a good job already centering content or if
+titles and logos are expected to appear in the center of the frame. It may be
+less useful on raw content where objects are not already well positioned on
+screen.
+
 ### Visualization to Facilitate Debugging
 
 `SceneCroppingCalculator` provides two extra output streams
@@ -307,7 +327,7 @@ node {
         score_aggregation_type: CONSTANT
       }
       scene_camera_motion_analyzer_options: {
-        motion_stabilization_threshold_percent: 0.3
+        motion_stabilization_threshold_percent: 0.5
         salient_point_bound: 0.499
       }
       padding_parameters: {

mediapipe/docs/examples.md

@@ -120,7 +120,7 @@ and do the model inference with the baseline model.
 MediaPipe for media processing to prepare video data sets for training a
 TensorFlow model.
 
-### Automatic video cropping
+### AutoFlip - Automatic video cropping
 
 [AutoFlip](./autoflip.md) shows how to use MediaPipe to build an automatic video
 cropping pipeline that can convert an input video to arbitrary aspect ratios.
@@ -142,6 +142,7 @@ GPU with live video from a webcam.
 *   [Desktop GPU](./face_detection_desktop.md)
 *   [Desktop CPU](./face_detection_desktop.md)
 
+
 ### Hand Tracking on Desktop with Webcam
 
 [Hand Tracking on Desktop with Webcam](./hand_tracking_desktop.md) shows how to
@@ -184,3 +185,18 @@ EdgeTPU on
 [Face Detection on Coral with Webcam](./face_detection_coral_devboard.md) shows
 how to use quantized face detection TFlite model accelerated with EdgeTPU on
 [Google Coral Dev Board](https://coral.withgoogle.com/products/dev-board).
+
+
+## Web Browser
+
+Below are samples that can directly be run in your web browser.
+See more details in [MediaPipe on the Web](./web.md) and
+[Google Developer blog post](https://mediapipe.page.link/webdevblog)
+
+### [Face Detection In Browser](https://viz.mediapipe.dev/demo/face_detection)
+
+### [Hand Detection In Browser](https://viz.mediapipe.dev/demo/hand_detection)
+
+### [Hand Tracking In Browser](https://viz.mediapipe.dev/demo/hand_tracking)
+
+### [Hair Segmentation In Browser](https://viz.mediapipe.dev/demo/hair_segmentation)

mediapipe/docs/face_detection_desktop.md

@@ -18,7 +18,9 @@ Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
 see
 [step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
 
-Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+Note: If MediaPipe depends on OpenCV 2, please see the
+[known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
+section.
 
 ### TensorFlow Lite Face Detection Demo with Webcam (CPU)
 
@@ -66,6 +68,9 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_detection/face_de
     --calculator_graph_config_file=mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt
 ```
 
+Issues running? Please first
+[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
+
 #### Graph
 
 ![graph visualization](images/face_detection_desktop.png)

mediapipe/docs/gpu.md

@@ -2,6 +2,7 @@
 
 -   [Overview](#overview)
 -   [OpenGL Support](#opengl-support)
+-   [Desktop GPUs](#desktop-gpu-linux)
 -   [Life of a GPU calculator](#life-of-a-gpu-calculator)
 -   [GpuBuffer to ImageFrame converters](#gpubuffer-to-imageframe-converters)
 -   [Disable GPU support](#disable-gpu-support)
@@ -22,13 +23,60 @@ Below are the design principles for GPU support in MediaPipe
    * Because different platforms may require different techniques for best performance, the API should allow flexibility in the way things are implemented behind the scenes.
    * A calculator should be allowed maximum flexibility in using the GPU for all or part of its operation, combining it with the CPU if necessary.
 
-### OpenGL support
+### OpenGL Support
 
-MediaPipe supports OpenGL ES up to version 3.2 on Android and up to ES 3.0 on iOS. In addition, MediaPipe also supports Metal on iOS.
+MediaPipe supports OpenGL ES up to version 3.2 on Android/Linux and up to ES 3.0
+on iOS. In addition, MediaPipe also supports Metal on iOS.
 
-   * MediaPipe allows graphs to run OpenGL in multiple GL contexts. For example, this can be very useful in graphs that combine a slower GPU inference path (eg, at 10 FPS) with a faster GPU rendering path (eg, at 30 FPS): since one GL context corresponds to one sequential command queue, using the same context for both tasks would reduce the rendering frame rate. One challenge MediaPipe's use of multiple contexts solves is the ability to communicate across them. An example scenario is one with an input video that is sent to both the rendering and inferences paths, and rendering needs to have access to the latest output from inference.
+OpenGL ES 3.1 or greater is required (on Android/Linux systems) for running
+machine learning inference calculators and graphs.
 
-   * An OpenGL context cannot be accessed by multiple threads at the same time. Furthermore, switching the active GL context on the same thread can be slow on some Android devices. Therefore, our approach is to have one dedicated thread per context. Each thread issues GL commands, building up a serial command queue on its context, which is then executed by the GPU asynchronously.
+MediaPipe allows graphs to run OpenGL in multiple GL contexts. For example, this
+can be very useful in graphs that combine a slower GPU inference path (eg, at 10
+FPS) with a faster GPU rendering path (eg, at 30 FPS): since one GL context
+corresponds to one sequential command queue, using the same context for both
+tasks would reduce the rendering frame rate.
+
+One challenge MediaPipe's use of multiple contexts solves is the ability to
+communicate across them. An example scenario is one with an input video that is
+sent to both the rendering and inferences paths, and rendering needs to have
+access to the latest output from inference.
+
+An OpenGL context cannot be accessed by multiple threads at the same time.
+Furthermore, switching the active GL context on the same thread can be slow on
+some Android devices. Therefore, our approach is to have one dedicated thread
+per context. Each thread issues GL commands, building up a serial command queue
+on its context, which is then executed by the GPU asynchronously.
+
+#### Desktop GPU (Linux)
+
+MediaPipe GPU can run on linux systems with video cards that support OpenGL ES
+3.1 and up.
+
+To check if your linux desktop GPU can run mediapipe:
+
+```bash
+$ sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
+$ sudo apt-get install mesa-utils
+$ glxinfo | grep -i opengl
+```
+
+My linux box prints:
+
+```bash
+$ glxinfo | grep -i opengl
+...
+OpenGL ES profile version string: OpenGL ES 3.2 NVIDIA 430.50
+OpenGL ES profile shading language version string: OpenGL ES GLSL ES 3.20
+OpenGL ES profile extensions:
+```
+
+*^notice the OpenGL ES 3.2 text^*
+
+To run MediaPipe GPU on desktop, you need to see ES 3.1 or greater printed.
+
+If OpenGL ES is not printed, or is below 3.1, then the GPU inference will not
+run.
 
 ### Life of a GPU calculator
 

mediapipe/docs/hair_segmentation_desktop.md

@@ -15,7 +15,9 @@ Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
 see
 [step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
 
-Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+Note: If MediaPipe depends on OpenCV 2, please see the
+[known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
+section.
 
 ### TensorFlow Lite Hair Segmentation Demo with Webcam (GPU)
 
@@ -40,6 +42,9 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hair_segmentation/hair
     --calculator_graph_config_file=mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt
 ```
 
+Issues running? Please first
+[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
+
 #### Graph
 
 ![hair_segmentation_mobile_gpu_graph](images/mobile/hair_segmentation_mobile_gpu.png)

mediapipe/docs/hand_tracking_desktop.md

@@ -17,7 +17,9 @@ Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
 see
 [step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
 
-Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+Note: If MediaPipe depends on OpenCV 2, please see the
+[known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
+section.
 
 ### TensorFlow Lite Hand Tracking Demo with Webcam (CPU)
 
@@ -61,6 +63,9 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tra
     --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt
 ```
 
+Issues running? Please first
+[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
+
 #### Graph
 
 ![graph visualization](images/hand_tracking_desktop.png)

mediapipe/docs/install.md

@@ -7,8 +7,9 @@ future.
 Note: If you plan to use TensorFlow calculators and example apps, there is a
 known issue with gcc and g++ version 6.3 and 7.3. Please use other versions.
 
-Note: To make Mediapipe work with TensorFlow, please install the python "future"
-library and the python "six" library using `pip install --user future six`.
+Note: To make Mediapipe work with TensorFlow, please set Python 3.7 as the
+default Python version and install the Python "six" library by running `pip3
+install --user six`.
 
 Choose your operating system:
 
@@ -230,7 +231,7 @@ To build and run iOS apps:
 
     *   Install [Homebrew](https://brew.sh).
     *   Install [Xcode](https://developer.apple.com/xcode/) and its Command Line
-        Tools by `xcode-select install`.
+        Tools by `xcode-select --install`.
 
 2.  Checkout MediaPipe repository.
 
@@ -265,12 +266,10 @@ To build and run iOS apps:
 
     ```bash
     $ brew install opencv@3
-    ```
 
-    Note: If you do `$brew install opencv`, there is a known issue caused by the
-    glog dependency of OpenCV 4.1.1 or above. The problem is solvable by
-    uninstalling the glog. You need to do `$ brew uninstall
-    --ignore-dependencies glog`
+    # There is a known issue caused by the glog dependency. Uninstall glog.
+    $ brew uninstall --ignore-dependencies glog
+    ```
 
     Option 2. Use MacPorts package manager tool to install the OpenCV libraries.
 
@@ -332,7 +331,17 @@ To build and run iOS apps:
 
     ```
 
-5.  Run the [Hello World desktop example](./hello_world_desktop.md).
+5.  Make sure that Python 3 and the Python "six" library are installed.
+
+    ```
+    $ brew install python
+    $ sudo ln -s -f /usr/local/bin/python3.7 /usr/local/bin/python
+    $ python --version
+    Python 3.7.4
+    $ pip3 install --user six
+    ```
+
+6.  Run the [Hello World desktop example](./hello_world_desktop.md).
 
     ```bash
     $ export GLOG_logtostderr=1
@@ -565,6 +574,7 @@ This will use a Docker image that will isolate mediapipe's installation from the
 
 Requirements:
 
+*   Java Runtime.
 *   Android SDK release 28.0.3 and above.
 *   Android NDK r17c and above.
 

mediapipe/docs/mediapipe_ios_setup.md

@@ -1,28 +1,32 @@
 ## Setting up MediaPipe for iOS
 
-1.  Install [Xcode](https://developer.apple.com/xcode/).
+1.  Install [Xcode](https://developer.apple.com/xcode/) and the Command Line
+    Tools.
 
-    Follow Apple's instructions to obtain the required developemnt certificates
-    and provisioning profiles for your iOS device.
+    Follow Apple's instructions to obtain the required development certificates
+    and provisioning profiles for your iOS device. Install the Command Line
+    Tools by
 
-2.  Install [Bazel](https://bazel.build/).
+    ```bash
+    xcode-select --install
+    ```
+
+2.  Install [Bazel 1.1.0](https://bazel.build/).
 
-    See their
-    [instructions](https://docs.bazel.build/versions/master/install-os-x.html).
     We recommend using [Homebrew](https://brew.sh/):
 
     ```bash
-    brew tap bazelbuild/tap
-    brew install bazelbuild/tap/bazel
+    $ brew install https://raw.githubusercontent.com/bazelbuild/homebrew-tap/f8a0fa981bcb1784a0d0823e14867b844e94fb3d/Formula/bazel.rb
     ```
 
-3.  Install python "future" and "six".
+3.  Set Python 3.7 as the default Python version and install the Python "six"
+    library.
 
-    To make Mediapipe work with TensorFlow, please install the python "future"
-    library and the python "six" library:
+    To make Mediapipe work with TensorFlow, please set Python 3.7 as the default
+    Python version and install the Python "six" library.
 
     ```bash
-    pip install --user future six
+    pip3 install --user six
     ```
 
 4.  Clone the MediaPipe repository.

mediapipe/docs/multi_hand_tracking_desktop.md

@@ -19,7 +19,8 @@ see
 [step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
 
 Note: If MediaPipe depends on OpenCV 2, please see the
-[known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+[known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
+section.
 
 ### TensorFlow Lite Multi-Hand Tracking Demo with Webcam (CPU)
 
@@ -61,6 +62,9 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/mu
     --calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt
 ```
 
+Issues running? Please first
+[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
+
 #### Graph
 
 ![graph visualization](images/multi_hand_tracking_desktop.png)

mediapipe/docs/object_detection_desktop.md

@@ -214,7 +214,6 @@ To build and run the TensorFlow Lite example on desktop (CPU) with Webcam, run:
 # Video from webcam running on desktop CPU
 $ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
     mediapipe/examples/desktop/object_detection:object_detection_cpu
-
 # It should print:
 #Target //mediapipe/examples/desktop/object_detection:object_detection_cpu up-to-date:
 #  bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_cpu

mediapipe/docs/web.md

@@ -7,6 +7,8 @@ in the browser client-side. The official API is under construction, but the core
 technology has been proven effective, and we can already show interactive
 cross-platform demos using your live webcam.
 
+[For more details, read this Google Developer blog post](https://mediapipe.page.link/webdevblog)
+
 ![image](images/web_effect.gif) ![image](images/web_segmentation.gif)
 
 ### Hand Tracking (with and without SIMD support)
@@ -21,6 +23,3 @@ support.  Below are two different versions of the
   1. WebAssembly MVP [demo](https://mediapipe.page.link/cds-ht) running around 5-8 frames per second on Desktop Chrome
 
   2. WebAssembly SIMD [demo](https://mediapipe.page.link/cds-ht-simd) running around 15-18 frames per second on *Canary* Chrome for Desktop, which must additionally be launched with the option `--js-flags="--experimental-wasm-simd"`
-
-
-NOTE: This page is a work-in-progress. More to come soon!

mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/MainActivity.java

@@ -149,12 +149,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java

@@ -180,12 +180,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java

@@ -158,12 +158,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java

@@ -151,12 +151,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/coral/BUILD

@@ -40,7 +40,7 @@ cc_library(
 # Demos
 
 cc_binary(
-    name = "object_detection_cpu",
+    name = "object_detection_tpu",
     deps = [
         "//mediapipe/examples/coral:demo_run_graph_main",
         "//mediapipe/graphs/object_detection:desktop_tflite_calculators",
@@ -48,7 +48,7 @@ cc_binary(
 )
 
 cc_binary(
-    name = "face_detection_cpu",
+    name = "face_detection_tpu",
     deps = [
         "//mediapipe/examples/coral:demo_run_graph_main",
         "//mediapipe/graphs/face_detection:desktop_tflite_calculators",

mediapipe/examples/coral/README.md

@@ -19,13 +19,13 @@ Docker container for building MediaPipe applications that run on Edge TPU.
 * (on coral device) prepare MediaPipe
 
         cd ~
-        sudo apt-get install git
+        sudo apt-get install -y git
         git clone https://github.com/google/mediapipe.git
         mkdir mediapipe/bazel-bin
 
 * (on coral device) install opencv 3.2
 
-        sudo apt-get update && apt-get install -y libopencv-dev
+        sudo apt-get update && sudo apt-get install -y libopencv-dev
 
 * (on coral device) find all opencv libs
 
@@ -78,7 +78,7 @@ Docker container for building MediaPipe applications that run on Edge TPU.
 
         return NULL;
 
-* Edit  /edgetpu/libedgetpu/BUILD
+* Edit /edgetpu/libedgetpu/BUILD
 
       to add this build target
 
@@ -90,9 +90,9 @@ Docker container for building MediaPipe applications that run on Edge TPU.
            visibility = ["//visibility:public"],
          )
 
-* Edit *tflite_inference_calculator.cc*  BUILD rules:
+* Edit /mediapipe/mediapipe/calculators/tflite/BUILD to change rules for *tflite_inference_calculator.cc*
 
-        sed -i 's/\":tflite_inference_calculator_cc_proto\",/\":tflite_inference_calculator_cc_proto\",\n\t\"@edgetpu\/\/:header\",\n\t\"@libedgetpu\/\/:lib\",/g' mediapipe/calculators/tflite/BUILD
+        sed -i 's/\":tflite_inference_calculator_cc_proto\",/\":tflite_inference_calculator_cc_proto\",\n\t\"@edgetpu\/\/:header\",\n\t\"@libedgetpu\/\/:lib\",/g' /mediapipe/mediapipe/calculators/tflite/BUILD
 
       The above command should add
 
@@ -105,37 +105,37 @@ Docker container for building MediaPipe applications that run on Edge TPU.
 
 * Object detection demo
 
-        bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:object_detection_cpu
+        bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:object_detection_tpu
 
- Copy object_detection_cpu binary to the MediaPipe checkout on the coral device
+ Copy object_detection_tpu binary to the MediaPipe checkout on the coral device
 
         # outside docker env, open new terminal on host machine #
         docker ps
-        docker cp <container-id>:/mediapipe/bazel-bin/mediapipe/examples/coral/object_detection_cpu /tmp/.
-        mdt push /tmp/object_detection_cpu /home/mendel/mediapipe/bazel-bin/.
+        docker cp <container-id>:/mediapipe/bazel-bin/mediapipe/examples/coral/object_detection_tpu /tmp/.
+        mdt push /tmp/object_detection_tpu /home/mendel/mediapipe/bazel-bin/.
 
 * Face detection demo
 
-        bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:face_detection_cpu
+        bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:face_detection_tpu
 
- Copy face_detection_cpu binary to the MediaPipe checkout on the coral device
+ Copy face_detection_tpu binary to the MediaPipe checkout on the coral device
 
         # outside docker env, open new terminal on host machine #
         docker ps
-        docker cp <container-id>:/mediapipe/bazel-bin/mediapipe/examples/coral/face_detection_cpu /tmp/.
-        mdt push /tmp/face_detection_cpu /home/mendel/mediapipe/bazel-bin/.
+        docker cp <container-id>:/mediapipe/bazel-bin/mediapipe/examples/coral/face_detection_tpu /tmp/.
+        mdt push /tmp/face_detection_tpu /home/mendel/mediapipe/bazel-bin/.
 
 ## On the coral device (with display)
 
      # Object detection
      cd ~/mediapipe
-     chmod +x bazel-bin/object_detection_cpu
+     chmod +x bazel-bin/object_detection_tpu
      export GLOG_logtostderr=1
-     bazel-bin/object_detection_cpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt
+     bazel-bin/object_detection_tpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt
 
      # Face detection
      cd ~/mediapipe
-     chmod +x bazel-bin/face_detection_cpu
+     chmod +x bazel-bin/face_detection_tpu
      export GLOG_logtostderr=1
-     bazel-bin/face_detection_cpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt
+     bazel-bin/face_detection_tpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt
 

mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt

@@ -1,6 +1,6 @@
-# MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
+# MediaPipe graph that performs face detection with TensorFlow Lite on TPU.
 # Used in the examples in
-# mediapipe/examples/coral:face_detection_cpu.
+# mediapipe/examples/coral:face_detection_tpu.
 
 # Images on GPU coming into and out of the graph.
 input_stream: "input_video"
@@ -36,7 +36,7 @@ node {
 node: {
   calculator: "ImageTransformationCalculator"
   input_stream: "IMAGE:throttled_input_video"
-  output_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "IMAGE:transformed_input_video"
   output_stream: "LETTERBOX_PADDING:letterbox_padding"
   options: {
     [mediapipe.ImageTransformationCalculatorOptions.ext] {
@@ -51,7 +51,7 @@ node: {
 # TfLiteTensor.
 node {
   calculator: "TfLiteConverterCalculator"
-  input_stream: "IMAGE:transformed_input_video_cpu"
+  input_stream: "IMAGE:transformed_input_video"
   output_stream: "TENSORS:image_tensor"
   options: {
     [mediapipe.TfLiteConverterCalculatorOptions.ext] {
@@ -60,7 +60,7 @@ node {
   }
 }
 
-# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# Runs a TensorFlow Lite model on TPU that takes an image tensor and outputs a
 # vector of tensors representing, for instance, detection boxes/keypoints and
 # scores.
 node {

mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt

@@ -1,8 +1,8 @@
-# MediaPipe graph that performs object detection with TensorFlow Lite on CPU.
+# MediaPipe graph that performs object detection with TensorFlow Lite on TPU.
 # Used in the examples in
-# mediapipie/examples/coral:object_detection_cpu.
+# mediapipie/examples/coral:object_detection_tpu.
 
-# Images on CPU coming into and out of the graph.
+# Images on TPU coming into and out of the graph.
 input_stream: "input_video"
 output_stream: "output_video"
 
@@ -30,7 +30,7 @@ node {
   output_stream: "throttled_input_video"
 }
 
-# Transforms the input image on CPU to a 320x320 image. To scale the image, by
+# Transforms the input image on CPU to a 300x300 image. To scale the image, by
 # default it uses the STRETCH scale mode that maps the entire input image to the
 # entire transformed image. As a result, image aspect ratio may be changed and
 # objects in the image may be deformed (stretched or squeezed), but the object
@@ -60,7 +60,7 @@ node {
   }
 }
 
-# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# Runs a TensorFlow Lite model on TPU that takes an image tensor and outputs a
 # vector of tensors representing, for instance, detection boxes/keypoints and
 # scores.
 node {

mediapipe/examples/desktop/autoflip/README.md

@@ -17,9 +17,7 @@
 
     GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/autoflip/run_autoflip \
       --calculator_graph_config_file=mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt \
-      --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,\
-        output_video_path=/absolute/path/to/save/the/output/video/file,\
-        aspect_ratio=width:height
+      --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,output_video_path=/absolute/path/to/save/the/output/video/file,aspect_ratio=width:height
     ```
 
 3.  View the cropped video.

mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt

@@ -35,7 +35,7 @@ node {
   options: {
     [mediapipe.PacketThinnerCalculatorOptions.ext]: {
       thinner_type: ASYNC
-      period: 500000
+      period: 200000
     }
   }
 }
@@ -165,7 +165,7 @@ node {
         score_aggregation_type: CONSTANT
       }
       scene_camera_motion_analyzer_options: {
-        motion_stabilization_threshold_percent: 0.3
+        motion_stabilization_threshold_percent: 0.5
         salient_point_bound: 0.499
       }
       padding_parameters: {

mediapipe/examples/desktop/autoflip/autoflip_graph_development.pbtxt

@@ -35,7 +35,7 @@ node {
   options: {
     [mediapipe.PacketThinnerCalculatorOptions.ext]: {
       thinner_type: ASYNC
-      period: 500000
+      period: 200000
     }
   }
 }
@@ -167,7 +167,7 @@ node {
         score_aggregation_type: CONSTANT
       }
       scene_camera_motion_analyzer_options: {
-        motion_stabilization_threshold_percent: 0.3
+        motion_stabilization_threshold_percent: 0.5
         salient_point_bound: 0.499
       }
       padding_parameters: {

mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.cc

@@ -52,15 +52,24 @@ struct PolynomialResidual {
   const double out_;
 };
 
+// Computes the amount of delta position change along the fitted polynomial
+// curve, translates the delta from being relative to the origin of the original
+// dimension to being relative to the center of the original dimension, then
+// regulates the delta to avoid moving camera off the frame boundaries.
 float ComputeDelta(const float in, const int original_dimension,
                    const int output_dimension, const double a, const double b,
                    const double c, const double d, const double k) {
+  // The value `out` here represents a normalized distance between the center of
+  // the output window and the origin of the original window.
   float out =
       a * in + b * in * in + c * in * in * in + d * in * in * in * in + k;
-  float delta = (out - 0.5) * 2 * output_dimension;
-  const float max_delta = (original_dimension - output_dimension) / 2.0f;
+  // Translate `out` to a pixel distance between the center of the output window
+  // and the center of the original window. This value can be negative, 0, or
+  // positive.
+  float delta = (out - 0.5) * original_dimension;
 
   // Make sure delta doesn't move the camera off the frame boundary.
+  const float max_delta = (original_dimension - output_dimension) / 2.0f;
   if (delta > max_delta) {
     delta = max_delta;
   } else if (delta < -max_delta) {

mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc

@@ -140,7 +140,7 @@ constexpr double prediction[] = {
     -24.550379,  -24.06503,
 };
 
-void GenerateDataPoints(
+void GenerateDataPointsFromRealVideo(
     const int focus_point_frames_length,
     const int prior_focus_point_frames_length,
     std::vector<FocusPointFrame>* focus_point_frames,
@@ -163,14 +163,15 @@ void GenerateDataPoints(
   }
 }
 
-TEST(PolynomialRegressionPathSolverTest, Success) {
+TEST(PolynomialRegressionPathSolverTest, SuccessInTrackingCameraMode) {
   PolynomialRegressionPathSolver solver;
   std::vector<FocusPointFrame> focus_point_frames;
   std::vector<FocusPointFrame> prior_focus_point_frames;
   std::vector<cv::Mat> all_xforms;
-  GenerateDataPoints(/* focus_point_frames_length = */ 100,
-                     /* prior_focus_point_frames_length = */ 100,
-                     &focus_point_frames, &prior_focus_point_frames);
+  GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 100,
+                                  /* prior_focus_point_frames_length = */ 100,
+                                  &focus_point_frames,
+                                  &prior_focus_point_frames);
   constexpr int kFrameWidth = 200;
   constexpr int kFrameHeight = 300;
   constexpr int kCropWidth = 100;
@@ -185,14 +186,86 @@ TEST(PolynomialRegressionPathSolverTest, Success) {
   }
 }
 
+TEST(PolynomialRegressionPathSolverTest, SuccessInStationaryCameraMode) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+
+  constexpr int kFocusPointFramesLength = 100;
+  constexpr float kNormStationaryFocusPointX = 0.34f;
+
+  for (int i = 0; i < kFocusPointFramesLength; i++) {
+    FocusPoint sp;
+    // Add a fixed normalized focus point location.
+    sp.set_norm_point_x(kNormStationaryFocusPointX);
+    FocusPointFrame spf;
+    *spf.add_point() = sp;
+    focus_point_frames.push_back(spf);
+  }
+  constexpr int kFrameWidth = 300;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  MP_ASSERT_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
+      kCropWidth, kCropHeight, &all_xforms));
+  ASSERT_EQ(all_xforms.size(), kFocusPointFramesLength);
+
+  constexpr int kExpectedShift = -48;
+  for (int i = 0; i < all_xforms.size(); i++) {
+    cv::Mat mat = all_xforms[i];
+    EXPECT_FLOAT_EQ(mat.at<float>(0, 2), kExpectedShift);
+  }
+}
+
+// Test the case where focus points are so close to boundaries that the amount
+// of shifts would have moved the camera to go outside frame boundaries. In this
+// case, the solver should regulate the camera position shift to keep it stay
+// inside the viewport.
+TEST(PolynomialRegressionPathSolverTest, SuccessWhenFocusPointCloseToBoundary) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+
+  constexpr int kFocusPointFramesLength = 100;
+  constexpr float kNormStationaryFocusPointX = 0.99f;
+
+  for (int i = 0; i < kFocusPointFramesLength; i++) {
+    FocusPoint sp;
+    // Add a fixed normalized focus point location.
+    sp.set_norm_point_x(kNormStationaryFocusPointX);
+    FocusPointFrame spf;
+    *spf.add_point() = sp;
+    focus_point_frames.push_back(spf);
+  }
+  constexpr int kFrameWidth = 500;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  MP_ASSERT_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
+      kCropWidth, kCropHeight, &all_xforms));
+  ASSERT_EQ(all_xforms.size(), kFocusPointFramesLength);
+
+  // Regulate max delta change = (500 - 100) / 2.
+  constexpr int kExpectedShift = 200;
+  for (int i = 0; i < all_xforms.size(); i++) {
+    cv::Mat mat = all_xforms[i];
+    EXPECT_FLOAT_EQ(mat.at<float>(0, 2), kExpectedShift);
+  }
+}
+
 TEST(PolynomialRegressionPathSolverTest, FewFramesShouldWork) {
   PolynomialRegressionPathSolver solver;
   std::vector<FocusPointFrame> focus_point_frames;
   std::vector<FocusPointFrame> prior_focus_point_frames;
   std::vector<cv::Mat> all_xforms;
-  GenerateDataPoints(/* focus_point_frames_length = */ 1,
-                     /* prior_focus_point_frames_length = */ 1,
-                     &focus_point_frames, &prior_focus_point_frames);
+  GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 1,
+                                  /* prior_focus_point_frames_length = */ 1,
+                                  &focus_point_frames,
+                                  &prior_focus_point_frames);
   constexpr int kFrameWidth = 200;
   constexpr int kFrameHeight = 300;
   constexpr int kCropWidth = 100;
@@ -208,9 +281,10 @@ TEST(PolynomialRegressionPathSolverTest, OneCurrentFrameShouldWork) {
   std::vector<FocusPointFrame> focus_point_frames;
   std::vector<FocusPointFrame> prior_focus_point_frames;
   std::vector<cv::Mat> all_xforms;
-  GenerateDataPoints(/* focus_point_frames_length = */ 1,
-                     /* prior_focus_point_frames_length = */ 0,
-                     &focus_point_frames, &prior_focus_point_frames);
+  GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 1,
+                                  /* prior_focus_point_frames_length = */ 0,
+                                  &focus_point_frames,
+                                  &prior_focus_point_frames);
   constexpr int kFrameWidth = 200;
   constexpr int kFrameHeight = 300;
   constexpr int kCropWidth = 100;
@@ -226,9 +300,10 @@ TEST(PolynomialRegressionPathSolverTest, ZeroFrameShouldFail) {
   std::vector<FocusPointFrame> focus_point_frames;
   std::vector<FocusPointFrame> prior_focus_point_frames;
   std::vector<cv::Mat> all_xforms;
-  GenerateDataPoints(/* focus_point_frames_length = */ 0,
-                     /* prior_focus_point_frames_length = */ 0,
-                     &focus_point_frames, &prior_focus_point_frames);
+  GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 0,
+                                  /* prior_focus_point_frames_length = */ 0,
+                                  &focus_point_frames,
+                                  &prior_focus_point_frames);
   constexpr int kFrameWidth = 200;
   constexpr int kFrameHeight = 300;
   constexpr int kCropWidth = 100;

mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py

@@ -25,7 +25,7 @@ import sys
 
 from absl import app
 import tensorflow.compat.v1 as tf
-from tensorflow.python.tools import freeze_graph
+from tensorflow.compat.v1.python.tools import freeze_graph
 
 BASE_DIR = '/tmp/mediapipe/'
 

mediapipe/framework/BUILD

@@ -536,6 +536,7 @@ cc_library(
         "//mediapipe/framework/profiler:graph_profiler",
         "//mediapipe/framework/stream_handler:default_input_stream_handler",
         "//mediapipe/framework/stream_handler:in_order_output_stream_handler",
+        "//mediapipe/framework/tool:name_util",
         "//mediapipe/framework/tool:status_util",
         "//mediapipe/framework/tool:tag_map",
         "//mediapipe/framework/tool:validate_name",
@@ -1268,6 +1269,7 @@ cc_library(
         "//mediapipe/framework/port:source_location",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:topologicalsorter",
+        "//mediapipe/framework/tool:name_util",
         "//mediapipe/framework/tool:status_util",
         "//mediapipe/framework/tool:subgraph_expansion",
         "//mediapipe/framework/tool:validate",

mediapipe/framework/calculator_context_manager.h

@@ -50,7 +50,7 @@ class CalculatorContextManager {
           setup_shards_callback);
 
   // Invoked by CalculatorNode::CleanupAfterRun().
-  void CleanupAfterRun() LOCKS_EXCLUDED(contexts_mutex_);
+  void CleanupAfterRun() ABSL_LOCKS_EXCLUDED(contexts_mutex_);
 
   // Returns true if the default calculator context has been initialized.
   bool HasDefaultCalculatorContext() const {
@@ -66,7 +66,7 @@ class CalculatorContextManager {
   // The input timestamp of the calculator context is returned in
   // *context_input_timestamp.
   CalculatorContext* GetFrontCalculatorContext(
-      Timestamp* context_input_timestamp) LOCKS_EXCLUDED(contexts_mutex_);
+      Timestamp* context_input_timestamp) ABSL_LOCKS_EXCLUDED(contexts_mutex_);
 
   // For sequential execution, returns a pointer to the default calculator
   // context. For parallel execution, creates or reuses a calculator context,
@@ -75,16 +75,16 @@ class CalculatorContextManager {
   // The ownership of the calculator context object isn't tranferred to the
   // caller.
   CalculatorContext* PrepareCalculatorContext(Timestamp input_timestamp)
-      LOCKS_EXCLUDED(contexts_mutex_);
+      ABSL_LOCKS_EXCLUDED(contexts_mutex_);
 
   // Removes the context with the smallest input timestamp from active_contexts_
   // and moves the calculator context to idle_contexts_. The caller must
   // guarantee that the output shards in the calculator context have been
   // propagated before calling this function.
-  void RecycleCalculatorContext() LOCKS_EXCLUDED(contexts_mutex_);
+  void RecycleCalculatorContext() ABSL_LOCKS_EXCLUDED(contexts_mutex_);
 
   // Returns true if active_contexts_ is non-empty.
-  bool HasActiveContexts() LOCKS_EXCLUDED(contexts_mutex_);
+  bool HasActiveContexts() ABSL_LOCKS_EXCLUDED(contexts_mutex_);
 
   int NumberOfContextTimestamps(
       const CalculatorContext& calculator_context) const {
@@ -135,10 +135,10 @@ class CalculatorContextManager {
   absl::Mutex contexts_mutex_;
   // A map from input timestamps to calculator contexts.
   std::map<Timestamp, std::unique_ptr<CalculatorContext>> active_contexts_
-      GUARDED_BY(contexts_mutex_);
+      ABSL_GUARDED_BY(contexts_mutex_);
   // Idle calculator contexts that are ready for reuse.
   std::deque<std::unique_ptr<CalculatorContext>> idle_contexts_
-      GUARDED_BY(contexts_mutex_);
+      ABSL_GUARDED_BY(contexts_mutex_);
 };
 
 }  // namespace mediapipe

mediapipe/framework/calculator_graph.cc

@@ -127,7 +127,13 @@ CalculatorGraph::CalculatorGraph(const CalculatorGraphConfig& config)
 // Defining the destructor here lets us use incomplete types in the header;
 // they only need to be fully visible here, where their destructor is
 // instantiated.
-CalculatorGraph::~CalculatorGraph() {}
+CalculatorGraph::~CalculatorGraph() {
+  // Stop periodic profiler output to ublock Executor destructors.
+  ::mediapipe::Status status = profiler()->Stop();
+  if (!status.ok()) {
+    LOG(ERROR) << "During graph destruction: " << status;
+  }
+}
 
 ::mediapipe::Status CalculatorGraph::InitializePacketGeneratorGraph(
     const std::map<std::string, Packet>& side_packets) {

mediapipe/framework/calculator_graph.h

@@ -298,7 +298,7 @@ class CalculatorGraph {
   // Callback when an error is encountered.
   // Adds the error to the vector of errors.
   void RecordError(const ::mediapipe::Status& error)
-      LOCKS_EXCLUDED(error_mutex_);
+      ABSL_LOCKS_EXCLUDED(error_mutex_);
 
   // Returns the maximum input stream queue size.
   int GetMaxInputStreamQueueSize();
@@ -339,13 +339,14 @@ class CalculatorGraph {
 
   // Returns true if this node or graph input stream is connected to
   // any input stream whose queue has hit maximum capacity.
-  bool IsNodeThrottled(int node_id) LOCKS_EXCLUDED(full_input_streams_mutex_);
+  bool IsNodeThrottled(int node_id)
+      ABSL_LOCKS_EXCLUDED(full_input_streams_mutex_);
 
   // If any active source node or graph input stream is throttled and not yet
   // closed, increases the max_queue_size for each full input stream in the
   // graph.
   // Returns true if at least one max_queue_size has been grown.
-  bool UnthrottleSources() LOCKS_EXCLUDED(full_input_streams_mutex_);
+  bool UnthrottleSources() ABSL_LOCKS_EXCLUDED(full_input_streams_mutex_);
 
   // Returns the scheduler's runtime measures for overhead measurement.
   // Only meant for test purposes.
@@ -498,7 +499,7 @@ class CalculatorGraph {
   // handler fails, it appends its error to errors_, and CleanupAfterRun sets
   // |*status| to the new combined errors on return.
   void CleanupAfterRun(::mediapipe::Status* status)
-      LOCKS_EXCLUDED(error_mutex_);
+      ABSL_LOCKS_EXCLUDED(error_mutex_);
 
   // Combines errors into a status. Returns true if the vector of errors is
   // non-empty.
@@ -571,7 +572,7 @@ class CalculatorGraph {
   // Mode for adding packets to a graph input stream. Set to block until all
   // affected input streams are not full by default.
   GraphInputStreamAddMode graph_input_stream_add_mode_
-      GUARDED_BY(full_input_streams_mutex_);
+      ABSL_GUARDED_BY(full_input_streams_mutex_);
 
   // For a source node or graph input stream (specified using id),
   // this stores the set of dependent input streams that have hit their
@@ -580,7 +581,7 @@ class CalculatorGraph {
   // is added to a graph input stream only if this set is empty.
   // Note that this vector contains an unused entry for each non-source node.
   std::vector<absl::flat_hash_set<InputStreamManager*>> full_input_streams_
-      GUARDED_BY(full_input_streams_mutex_);
+      ABSL_GUARDED_BY(full_input_streams_mutex_);
 
   // Maps stream names to graph input stream objects.
   absl::flat_hash_map<std::string, std::unique_ptr<GraphInputStream>>
@@ -606,7 +607,7 @@ class CalculatorGraph {
 
   // Vector of errors encountered while running graph. Always use RecordError()
   // to add an error to this vector.
-  std::vector<::mediapipe::Status> errors_ GUARDED_BY(error_mutex_);
+  std::vector<::mediapipe::Status> errors_ ABSL_GUARDED_BY(error_mutex_);
 
   // True if the default executor uses the application thread.
   bool use_application_thread_ = false;
@@ -614,7 +615,7 @@ class CalculatorGraph {
   // Condition variable that waits until all input streams that depend on a
   // graph input stream are below the maximum queue size.
   absl::CondVar wait_to_add_packet_cond_var_
-      GUARDED_BY(full_input_streams_mutex_);
+      ABSL_GUARDED_BY(full_input_streams_mutex_);
 
   // Mutex for the vector of errors.
   absl::Mutex error_mutex_;

mediapipe/framework/calculator_graph_event_loop_test.cc

@@ -44,7 +44,7 @@ class CalculatorGraphEventLoopTest : public testing::Test {
   }
 
  protected:
-  std::vector<Packet> output_packets_ GUARDED_BY(output_packets_mutex_);
+  std::vector<Packet> output_packets_ ABSL_GUARDED_BY(output_packets_mutex_);
   absl::Mutex output_packets_mutex_;
 };
 

mediapipe/framework/calculator_node.cc

@@ -42,6 +42,7 @@
 #include "mediapipe/framework/port/source_location.h"
 #include "mediapipe/framework/port/status_builder.h"
 #include "mediapipe/framework/timestamp.h"
+#include "mediapipe/framework/tool/name_util.h"
 #include "mediapipe/framework/tool/status_util.h"
 #include "mediapipe/framework/tool/tag_map.h"
 #include "mediapipe/framework/tool/validate_name.h"
@@ -85,7 +86,7 @@ Timestamp CalculatorNode::SourceProcessOrder(
 
   const CalculatorGraphConfig::Node& node_config =
       validated_graph_->Config().node(node_id_);
-  name_ = CanonicalNodeName(validated_graph_->Config(), node_id_);
+  name_ = tool::CanonicalNodeName(validated_graph_->Config(), node_id_);
 
   max_in_flight_ = node_config.max_in_flight();
   max_in_flight_ = max_in_flight_ ? max_in_flight_ : 1;

mediapipe/framework/calculator_node.h

@@ -128,25 +128,25 @@ class CalculatorNode {
       std::function<void()> source_node_opened_callback,
       std::function<void(CalculatorContext*)> schedule_callback,
       std::function<void(::mediapipe::Status)> error_callback,
-      CounterFactory* counter_factory) LOCKS_EXCLUDED(status_mutex_);
+      CounterFactory* counter_factory) ABSL_LOCKS_EXCLUDED(status_mutex_);
   // Opens the node.
-  ::mediapipe::Status OpenNode() LOCKS_EXCLUDED(status_mutex_);
+  ::mediapipe::Status OpenNode() ABSL_LOCKS_EXCLUDED(status_mutex_);
   // Called when a source node's layer becomes active.
-  void ActivateNode() LOCKS_EXCLUDED(status_mutex_);
+  void ActivateNode() ABSL_LOCKS_EXCLUDED(status_mutex_);
   // Cleans up the node after the CalculatorGraph has been run. Deletes
   // the Calculator managed by this node. graph_status is the status of
   // the graph run.
   void CleanupAfterRun(const ::mediapipe::Status& graph_status)
-      LOCKS_EXCLUDED(status_mutex_);
+      ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Returns true iff PrepareForRun() has been called (and types verified).
-  bool Prepared() const LOCKS_EXCLUDED(status_mutex_);
+  bool Prepared() const ABSL_LOCKS_EXCLUDED(status_mutex_);
   // Returns true iff Open() has been called on the calculator.
-  bool Opened() const LOCKS_EXCLUDED(status_mutex_);
+  bool Opened() const ABSL_LOCKS_EXCLUDED(status_mutex_);
   // Returns true iff a source calculator's layer is active.
-  bool Active() const LOCKS_EXCLUDED(status_mutex_);
+  bool Active() const ABSL_LOCKS_EXCLUDED(status_mutex_);
   // Returns true iff Close() has been called on the calculator.
-  bool Closed() const LOCKS_EXCLUDED(status_mutex_);
+  bool Closed() const ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Returns true iff this is a source node.
   //
@@ -166,32 +166,32 @@ class CalculatorNode {
   // then call EndScheduling when finished running it.
   // If false is returned, the scheduler must not execute the node.
   // This method is thread-safe.
-  bool TryToBeginScheduling() LOCKS_EXCLUDED(status_mutex_);
+  bool TryToBeginScheduling() ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Subtracts one from current_in_flight_ to allow a new invocation to be
   // scheduled. Then, it checks scheduling_state_ and invokes SchedulingLoop()
   // if necessary. This method is thread-safe.
   // TODO: this could be done implicitly by the call to ProcessNode
   // or CloseNode.
-  void EndScheduling() LOCKS_EXCLUDED(status_mutex_);
+  void EndScheduling() ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Returns true if OpenNode() can be scheduled.
-  bool ReadyForOpen() const LOCKS_EXCLUDED(status_mutex_);
+  bool ReadyForOpen() const ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Called by the InputStreamHandler when all the input stream headers
   // become available.
-  void InputStreamHeadersReady() LOCKS_EXCLUDED(status_mutex_);
+  void InputStreamHeadersReady() ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Called by the InputSidePacketHandler when all the input side packets
   // become available.
-  void InputSidePacketsReady() LOCKS_EXCLUDED(status_mutex_);
+  void InputSidePacketsReady() ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Checks scheduling_state_, and then invokes SchedulingLoop() if necessary.
   // This method is thread-safe.
-  void CheckIfBecameReady() LOCKS_EXCLUDED(status_mutex_);
+  void CheckIfBecameReady() ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Called by SchedulerQueue when a node is opened.
-  void NodeOpened() LOCKS_EXCLUDED(status_mutex_);
+  void NodeOpened() ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Returns whether this is a GPU calculator node.
   bool UsesGpu() const { return uses_gpu_; }
@@ -220,7 +220,7 @@ class CalculatorNode {
   // indicates whether the graph run has ended.
   ::mediapipe::Status CloseNode(const ::mediapipe::Status& graph_status,
                                 bool graph_run_ended)
-      LOCKS_EXCLUDED(status_mutex_);
+      ABSL_LOCKS_EXCLUDED(status_mutex_);
 
   // Returns a pointer to the default calculator context that is used for
   // sequential execution. A source node should always reuse its default
@@ -274,9 +274,9 @@ class CalculatorNode {
   void SchedulingLoop();
 
   // Closes the input and output streams.
-  void CloseInputStreams() LOCKS_EXCLUDED(status_mutex_);
+  void CloseInputStreams() ABSL_LOCKS_EXCLUDED(status_mutex_);
   void CloseOutputStreams(OutputStreamShardSet* outputs)
-      LOCKS_EXCLUDED(status_mutex_);
+      ABSL_LOCKS_EXCLUDED(status_mutex_);
   // Get a std::string describing the input streams.
   std::string DebugInputStreamNames() const;
 
@@ -304,7 +304,7 @@ class CalculatorNode {
     kStateActive = 3,
     kStateClosed = 4
   };
-  NodeStatus status_ GUARDED_BY(status_mutex_){kStateUninitialized};
+  NodeStatus status_ ABSL_GUARDED_BY(status_mutex_){kStateUninitialized};
 
   // The max number of invocations that can be scheduled in parallel.
   int max_in_flight_ = 1;
@@ -312,7 +312,7 @@ class CalculatorNode {
   // scheduling.
   //
   // The number of invocations that are scheduled but not finished.
-  int current_in_flight_ GUARDED_BY(status_mutex_) = 0;
+  int current_in_flight_ ABSL_GUARDED_BY(status_mutex_) = 0;
   // SchedulingState incidates the current state of the node scheduling process.
   // There are four possible transitions:
   // (a) From kIdle to kScheduling.
@@ -333,14 +333,15 @@ class CalculatorNode {
     kScheduling = 1,  //
     kSchedulingPending = 2
   };
-  SchedulingState scheduling_state_ GUARDED_BY(status_mutex_) = kIdle;
+  SchedulingState scheduling_state_ ABSL_GUARDED_BY(status_mutex_) = kIdle;
 
   std::function<void()> ready_for_open_callback_;
   std::function<void()> source_node_opened_callback_;
-  bool input_stream_headers_ready_called_ GUARDED_BY(status_mutex_) = false;
-  bool input_side_packets_ready_called_ GUARDED_BY(status_mutex_) = false;
-  bool input_stream_headers_ready_ GUARDED_BY(status_mutex_) = false;
-  bool input_side_packets_ready_ GUARDED_BY(status_mutex_) = false;
+  bool input_stream_headers_ready_called_ ABSL_GUARDED_BY(status_mutex_) =
+      false;
+  bool input_side_packets_ready_called_ ABSL_GUARDED_BY(status_mutex_) = false;
+  bool input_stream_headers_ready_ ABSL_GUARDED_BY(status_mutex_) = false;
+  bool input_side_packets_ready_ ABSL_GUARDED_BY(status_mutex_) = false;
 
   // Owns and manages all CalculatorContext objects.
   CalculatorContextManager calculator_context_manager_;

mediapipe/framework/calculator_parallel_execution_test.cc

@@ -85,7 +85,7 @@ class ParallelExecutionTest : public testing::Test {
   }
 
  protected:
-  std::vector<Packet> output_packets_ GUARDED_BY(output_packets_mutex_);
+  std::vector<Packet> output_packets_ ABSL_GUARDED_BY(output_packets_mutex_);
   absl::Mutex output_packets_mutex_;
 };
 

mediapipe/framework/counter_factory.cc

@@ -29,35 +29,35 @@ class BasicCounter : public Counter {
  public:
   explicit BasicCounter(const std::string& name) : value_(0) {}
 
-  void Increment() LOCKS_EXCLUDED(mu_) override {
+  void Increment() ABSL_LOCKS_EXCLUDED(mu_) override {
     absl::WriterMutexLock lock(&mu_);
     ++value_;
   }
 
-  void IncrementBy(int amount) LOCKS_EXCLUDED(mu_) override {
+  void IncrementBy(int amount) ABSL_LOCKS_EXCLUDED(mu_) override {
     absl::WriterMutexLock lock(&mu_);
     value_ += amount;
   }
 
-  int64 Get() LOCKS_EXCLUDED(mu_) override {
+  int64 Get() ABSL_LOCKS_EXCLUDED(mu_) override {
     absl::ReaderMutexLock lock(&mu_);
     return value_;
   }
 
  private:
   absl::Mutex mu_;
-  int64 value_ GUARDED_BY(mu_);
+  int64 value_ ABSL_GUARDED_BY(mu_);
 };
 
 }  // namespace
 
 CounterSet::CounterSet() {}
 
-CounterSet::~CounterSet() LOCKS_EXCLUDED(mu_) { PublishCounters(); }
+CounterSet::~CounterSet() ABSL_LOCKS_EXCLUDED(mu_) { PublishCounters(); }
 
-void CounterSet::PublishCounters() LOCKS_EXCLUDED(mu_) {}
+void CounterSet::PublishCounters() ABSL_LOCKS_EXCLUDED(mu_) {}
 
-void CounterSet::PrintCounters() LOCKS_EXCLUDED(mu_) {
+void CounterSet::PrintCounters() ABSL_LOCKS_EXCLUDED(mu_) {
   absl::ReaderMutexLock lock(&mu_);
   LOG_IF(INFO, !counters_.empty()) << "MediaPipe Counters:";
   for (const auto& counter : counters_) {
@@ -65,7 +65,7 @@ void CounterSet::PrintCounters() LOCKS_EXCLUDED(mu_) {
   }
 }
 
-Counter* CounterSet::Get(const std::string& name) LOCKS_EXCLUDED(mu_) {
+Counter* CounterSet::Get(const std::string& name) ABSL_LOCKS_EXCLUDED(mu_) {
   absl::ReaderMutexLock lock(&mu_);
   if (!::mediapipe::ContainsKey(counters_, name)) {
     return nullptr;
@@ -74,7 +74,7 @@ Counter* CounterSet::Get(const std::string& name) LOCKS_EXCLUDED(mu_) {
 }
 
 std::map<std::string, int64> CounterSet::GetCountersValues()
-    LOCKS_EXCLUDED(mu_) {
+    ABSL_LOCKS_EXCLUDED(mu_) {
   absl::ReaderMutexLock lock(&mu_);
   std::map<std::string, int64> result;
   for (const auto& it : counters_) {

mediapipe/framework/counter_factory.h

@@ -51,7 +51,7 @@ class CounterSet {
   // to the existing pointer.
   template <typename CounterType, typename... Args>
   Counter* Emplace(const std::string& name, Args&&... args)
-      LOCKS_EXCLUDED(mu_) {
+      ABSL_LOCKS_EXCLUDED(mu_) {
     absl::WriterMutexLock lock(&mu_);
     std::unique_ptr<Counter>* existing_counter = FindOrNull(counters_, name);
     if (existing_counter) {
@@ -66,11 +66,12 @@ class CounterSet {
   Counter* Get(const std::string& name);
 
   // Retrieves all counters names and current values from the internal map.
-  std::map<std::string, int64> GetCountersValues() LOCKS_EXCLUDED(mu_);
+  std::map<std::string, int64> GetCountersValues() ABSL_LOCKS_EXCLUDED(mu_);
 
  private:
   absl::Mutex mu_;
-  std::map<std::string, std::unique_ptr<Counter>> counters_ GUARDED_BY(mu_);
+  std::map<std::string, std::unique_ptr<Counter>> counters_
+      ABSL_GUARDED_BY(mu_);
 };
 
 // Generic counter factory

mediapipe/framework/deps/BUILD

@@ -175,6 +175,7 @@ cc_library(
     name = "random",
     hdrs = ["random_base.h"],
     visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework/port:integral_types"],
 )
 
 cc_library(

mediapipe/framework/deps/monotonic_clock.cc

@@ -33,7 +33,7 @@ struct MonotonicClock::State {
   Clock* raw_clock;
   absl::Mutex lock;
   // The largest time ever returned by Now().
-  absl::Time max_time GUARDED_BY(lock);
+  absl::Time max_time ABSL_GUARDED_BY(lock);
   explicit State(Clock* clock)
       : raw_clock(clock), max_time(absl::UnixEpoch()) {}
 };
@@ -171,13 +171,13 @@ class MonotonicClockImpl : public MonotonicClock {
   // last_raw_time_ remembers the last value obtained from raw_clock_.
   // It prevents spurious calls to ReportCorrection when time moves
   // forward by a smaller amount than a prior backward jump.
-  absl::Time last_raw_time_ GUARDED_BY(state_->lock);
+  absl::Time last_raw_time_ ABSL_GUARDED_BY(state_->lock);
 
   // Variables that keep track of time corrections made by this instance of
   // MonotonicClock.  (All such metrics are instance-local for reasons
   // described earlier.)
-  int correction_count_ GUARDED_BY(state_->lock);
-  absl::Duration max_correction_ GUARDED_BY(state_->lock);
+  int correction_count_ ABSL_GUARDED_BY(state_->lock);
+  absl::Duration max_correction_ ABSL_GUARDED_BY(state_->lock);
 };
 
 // Factory methods.

mediapipe/framework/deps/monotonic_clock_test.cc

@@ -456,7 +456,7 @@ class ClockFrenzy {
 
   // Provide a lock to avoid race conditions in non-threadsafe ACMRandom.
   mutable absl::Mutex lock_;
-  std::unique_ptr<RandomEngine> random_ GUARDED_BY(lock_);
+  std::unique_ptr<RandomEngine> random_ ABSL_GUARDED_BY(lock_);
 
   // The stopping notification.
   bool running_;

mediapipe/framework/deps/random_base.h

@@ -15,6 +15,8 @@
 #ifndef MEDIAPIPE_DEPS_RANDOM_BASE_H_
 #define MEDIAPIPE_DEPS_RANDOM_BASE_H_
 
+#include "mediapipe/framework/port/integral_types.h"
+
 class RandomBase {
  public:
   // constructors.  Don't do too much.
@@ -22,7 +24,8 @@ class RandomBase {
   virtual ~RandomBase();
 
   virtual float RandFloat() { return 0; }
-  virtual int UnbiasedUniform(int n) { return n - 1; }
+  virtual int UnbiasedUniform(int n) { return 0; }
+  virtual uint64 UnbiasedUniform64(uint64 n) { return 0; }
 };
 
 #endif  // MEDIAPIPE_DEPS_RANDOM_BASE_H_

mediapipe/framework/deps/registration.h

@@ -159,7 +159,7 @@ class FunctionRegistry {
   FunctionRegistry& operator=(const FunctionRegistry&) = delete;
 
   RegistrationToken Register(const std::string& name, Function func)
-      LOCKS_EXCLUDED(lock_) {
+      ABSL_LOCKS_EXCLUDED(lock_) {
     std::string normalized_name = GetNormalizedName(name);
     absl::WriterMutexLock lock(&lock_);
     std::string adjusted_name = GetAdjustedName(normalized_name);
@@ -189,7 +189,7 @@ class FunctionRegistry {
                                                   std::tuple<Args...>>::value,
                               int> = 0>
   ReturnType Invoke(const std::string& name, Args2&&... args)
-      LOCKS_EXCLUDED(lock_) {
+      ABSL_LOCKS_EXCLUDED(lock_) {
     Function function;
     {
       absl::ReaderMutexLock lock(&lock_);
@@ -207,14 +207,14 @@ class FunctionRegistry {
   // Namespaces in |name| and |ns| are separated by kNameSep.
   template <typename... Args2>
   ReturnType Invoke(const std::string& ns, const std::string& name,
-                    Args2&&... args) LOCKS_EXCLUDED(lock_) {
+                    Args2&&... args) ABSL_LOCKS_EXCLUDED(lock_) {
     return Invoke(GetQualifiedName(ns, name), args...);
   }
 
   // Note that it's possible for registered implementations to be subsequently
   // unregistered, though this will never happen with registrations made via
   // MEDIAPIPE_REGISTER_FACTORY_FUNCTION.
-  bool IsRegistered(const std::string& name) const LOCKS_EXCLUDED(lock_) {
+  bool IsRegistered(const std::string& name) const ABSL_LOCKS_EXCLUDED(lock_) {
     absl::ReaderMutexLock lock(&lock_);
     return functions_.count(name) != 0;
   }
@@ -222,7 +222,7 @@ class FunctionRegistry {
   // Returns true if the specified factory function is available.
   // Namespaces in |name| and |ns| are separated by kNameSep.
   bool IsRegistered(const std::string& ns, const std::string& name) const
-      LOCKS_EXCLUDED(lock_) {
+      ABSL_LOCKS_EXCLUDED(lock_) {
     return IsRegistered(GetQualifiedName(ns, name));
   }
 
@@ -231,7 +231,7 @@ class FunctionRegistry {
   // unregistered, though this will never happen with registrations made via
   // MEDIAPIPE_REGISTER_FACTORY_FUNCTION.
   std::unordered_set<std::string> GetRegisteredNames() const
-      LOCKS_EXCLUDED(lock_) {
+      ABSL_LOCKS_EXCLUDED(lock_) {
     absl::ReaderMutexLock lock(&lock_);
     std::unordered_set<std::string> names;
     std::for_each(functions_.cbegin(), functions_.cend(),
@@ -287,7 +287,7 @@ class FunctionRegistry {
 
  private:
   mutable absl::Mutex lock_;
-  std::unordered_map<std::string, Function> functions_ GUARDED_BY(lock_);
+  std::unordered_map<std::string, Function> functions_ ABSL_GUARDED_BY(lock_);
 
   // For names included in NamespaceWhitelist, strips the namespace.
   std::string GetAdjustedName(const std::string& name) {

mediapipe/framework/deps/threadpool.h

@@ -93,8 +93,8 @@ class ThreadPool {
 
   absl::Mutex mutex_;
   absl::CondVar condition_;
-  bool stopped_ GUARDED_BY(mutex_) = false;
-  std::deque<std::function<void()>> tasks_ GUARDED_BY(mutex_);
+  bool stopped_ ABSL_GUARDED_BY(mutex_) = false;
+  std::deque<std::function<void()>> tasks_ ABSL_GUARDED_BY(mutex_);
 
   ThreadOptions thread_options_;
 };

mediapipe/framework/encode_binary_proto.bzl

@@ -87,6 +87,7 @@ def _encode_binary_proto_impl(ctx):
                 ctx.executable._proto_compiler.path,
                 "--encode=" + ctx.attr.message_type,
                 "--proto_path=" + ctx.genfiles_dir.path,
+                "--proto_path=" + ctx.bin_dir.path,
                 "--proto_path=.",
             ] + path_list + file_list +
             ["<", textpb.path, ">", binarypb.path],
@@ -136,6 +137,7 @@ def _generate_proto_descriptor_set_impl(ctx):
         arguments = [
                         "--descriptor_set_out=%s" % descriptor.path,
                         "--proto_path=" + ctx.genfiles_dir.path,
+                        "--proto_path=" + ctx.bin_dir.path,
                         "--proto_path=.",
                     ] +
                     [s.path for s in all_protos.to_list()],

mediapipe/framework/formats/image_frame.cc

@@ -371,11 +371,12 @@ void ImageFrame::CopyPixelData(ImageFormat::Format format, int width,
 void ImageFrame::CopyToBuffer(uint8* buffer, int buffer_size) const {
   CHECK(buffer);
   CHECK_EQ(1, ByteDepth());
-  int data_size = width_ * height_ * NumberOfChannels() * ByteDepth();
+  const int data_size = width_ * height_ * NumberOfChannels();
   CHECK_LE(data_size, buffer_size);
   if (IsContiguous()) {
     // The data is stored contiguously, we can just copy.
-    std::copy_n(pixel_data_.get(), data_size, buffer);
+    const uint8* src = reinterpret_cast<const uint8*>(pixel_data_.get());
+    std::copy_n(src, data_size, buffer);
   } else {
     InternalCopyToBuffer(0 /* contiguous storage */,
                          reinterpret_cast<char*>(buffer));

mediapipe/framework/formats/rect.proto

@@ -29,6 +29,9 @@ message Rect {
 
   // Rotation angle is counter-clockwise in radian.
   optional float rotation = 5 [default = 0.0];
+
+  // Optional unique id to help associate different Rects to each other.
+  optional int64 rect_id = 6;
 }
 
 // A rectangle with rotation in normalized coordinates. The values of box center
@@ -45,4 +48,8 @@ message NormalizedRect {
 
   // Rotation angle is counter-clockwise in radian.
   optional float rotation = 5 [default = 0.0];
+
+  // Optional unique id to help associate different NormalizedRects to each
+  // other.
+  optional int64 rect_id = 6;
 }

mediapipe/framework/graph_output_stream.h

@@ -163,8 +163,8 @@ class OutputStreamPollerImpl : public GraphOutputStream {
 
  private:
   absl::Mutex mutex_;
-  absl::CondVar handler_condvar_ GUARDED_BY(mutex_);
-  bool graph_has_error_ GUARDED_BY(mutex_);
+  absl::CondVar handler_condvar_ ABSL_GUARDED_BY(mutex_);
+  bool graph_has_error_ ABSL_GUARDED_BY(mutex_);
 };
 
 }  // namespace internal

mediapipe/framework/graph_validation_test.cc

@@ -94,6 +94,7 @@ TEST(GraphValidationTest, InitializeGraphFromProtos) {
         }
         node {
           calculator: "PassThroughCalculator"
+          name: "passthroughgraph__PassThroughCalculator"
           input_stream: "stream_2"
           output_stream: "stream_3"
         }
@@ -201,7 +202,7 @@ TEST(GraphValidationTest, InitializeTemplateFromProtos) {
           output_stream: "stream_2"
         }
         node {
-          name: "__sg0_stream_8"
+          name: "passthroughgraph__stream_8"
           calculator: "PassThroughCalculator"
           input_stream: "stream_2"
           output_stream: "stream_3"
@@ -263,6 +264,7 @@ TEST(GraphValidationTest, OptionalSubgraphStreams) {
         }
         node {
           calculator: "PassThroughCalculator"
+          name: "passthroughgraph__PassThroughCalculator"
           input_stream: "foo_bar"
           output_stream: "foo_out"
         }
@@ -379,6 +381,7 @@ TEST(GraphValidationTest, OptionalInputNotProvidedForSubgraphCalculator) {
         output_stream: "OUTPUT:foo_out"
         node {
           calculator: "OptionalSideInputTestCalculator"
+          name: "passthroughgraph__OptionalSideInputTestCalculator"
           output_stream: "OUTPUT:foo_out"
         }
         executor {}

mediapipe/framework/input_stream_manager.cc

@@ -233,7 +233,7 @@ Timestamp InputStreamManager::MinTimestampOrBound(bool* is_empty) const {
 }
 
 Timestamp InputStreamManager::MinTimestampOrBoundHelper() const
-    EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_) {
+    ABSL_EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_) {
   return queue_.empty() ? next_timestamp_bound_ : queue_.front().Timestamp();
 }
 

mediapipe/framework/input_stream_manager.h

@@ -73,7 +73,7 @@ class InputStreamManager {
 
   // Reset the input stream for another run of the graph (i.e. another
   // image/video/audio).
-  void PrepareForRun() LOCKS_EXCLUDED(stream_mutex_);
+  void PrepareForRun() ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Adds a list of timestamped packets. Sets "notify" to true if the queue
   // becomes non-empty. Does nothing if the input stream is closed.
@@ -96,7 +96,7 @@ class InputStreamManager {
   ::mediapipe::Status MovePackets(std::list<Packet>* container, bool* notify);
 
   // Closes the input stream.  This function can be called multiple times.
-  void Close() LOCKS_EXCLUDED(stream_mutex_);
+  void Close() ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Sets the bound on the next timestamp to be added to the input stream.
   // Sets "notify" to true if the bound is advanced while the packet queue is
@@ -104,24 +104,24 @@ class InputStreamManager {
   // DisableTimestamps() is called. Does nothing if the input stream is
   // closed.
   ::mediapipe::Status SetNextTimestampBound(Timestamp bound, bool* notify)
-      LOCKS_EXCLUDED(stream_mutex_);
+      ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Returns the smallest timestamp at which we might see an input in
   // this input stream. This is the timestamp of the first item in the queue if
   // the queue is non-empty, or the next timestamp bound if it is empty.
   // Sets is_empty to queue_.empty() if it is not nullptr.
   Timestamp MinTimestampOrBound(bool* is_empty) const
-      LOCKS_EXCLUDED(stream_mutex_);
+      ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Turns off the use of packet timestamps.
   void DisableTimestamps();
 
   // Returns true iff the queue is empty.
-  bool IsEmpty() const LOCKS_EXCLUDED(stream_mutex_);
+  bool IsEmpty() const ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // If the queue is not empty, returns the packet at the front of the queue.
   // Otherwise, returns an empty packet.
-  Packet QueueHead() const LOCKS_EXCLUDED(stream_mutex_);
+  Packet QueueHead() const ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Advances time to timestamp.  Pops and returns the packet in the queue
   // with a matching timestamp, if it exists.  Time can be advanced to any
@@ -134,26 +134,26 @@ class InputStreamManager {
   // Timestamp::Done() after the pop.
   Packet PopPacketAtTimestamp(Timestamp timestamp, int* num_packets_dropped,
                               bool* stream_is_done)
-      LOCKS_EXCLUDED(stream_mutex_);
+      ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Pops and returns the packet at the head of the queue if the queue is
   // non-empty. Sets "stream_is_done" if  the next timestamp bound reaches
   // Timestamp::Done() after the pop.
-  Packet PopQueueHead(bool* stream_is_done) LOCKS_EXCLUDED(stream_mutex_);
+  Packet PopQueueHead(bool* stream_is_done) ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Returns the number of packets in the queue.
-  int QueueSize() const LOCKS_EXCLUDED(stream_mutex_);
+  int QueueSize() const ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Returns true iff the queue is full.
-  bool IsFull() const LOCKS_EXCLUDED(stream_mutex_);
+  bool IsFull() const ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Returns the max queue size. -1 indicates that there is no maximum.
-  int MaxQueueSize() const LOCKS_EXCLUDED(stream_mutex_);
+  int MaxQueueSize() const ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Sets the maximum queue size for the stream. Used to determine when the
   // callbacks for becomes_full and becomes_not_full should be invoked. A value
   // of -1 means that there is no maximum queue size.
-  void SetMaxQueueSize(int max_queue_size) LOCKS_EXCLUDED(stream_mutex_);
+  void SetMaxQueueSize(int max_queue_size) ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // If there are equal to or more than n packets in the queue, this function
   // returns the min timestamp of among the latest n packets of the queue.  If
@@ -161,12 +161,12 @@ class InputStreamManager {
   // Timestamp::Unset().
   // NOTE: This is a public API intended for FixedSizeInputStreamHandler only.
   Timestamp GetMinTimestampAmongNLatest(int n) const
-      LOCKS_EXCLUDED(stream_mutex_);
+      ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // pop_front()s packets that are earlier than the given timestamp.
   // NOTE: This is a public API intended for FixedSizeInputStreamHandler only.
   void ErasePacketsEarlierThan(Timestamp timestamp)
-      LOCKS_EXCLUDED(stream_mutex_);
+      ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // If a maximum queue size is specified (!= -1), these callbacks that are
   // invoked when the input queue becomes full (>= max_queue_size_) or when it
@@ -184,24 +184,24 @@ class InputStreamManager {
   template <typename Container>
   ::mediapipe::Status AddOrMovePacketsInternal(Container container,
                                                bool* notify)
-      LOCKS_EXCLUDED(stream_mutex_);
+      ABSL_LOCKS_EXCLUDED(stream_mutex_);
 
   // Returns true if the next timestamp bound reaches Timestamp::Done().
-  bool IsDone() const EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_);
+  bool IsDone() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_);
 
   // Returns the smallest timestamp at which this stream might see an input.
   Timestamp MinTimestampOrBoundHelper() const;
 
   mutable absl::Mutex stream_mutex_;
-  std::deque<Packet> queue_ GUARDED_BY(stream_mutex_);
+  std::deque<Packet> queue_ ABSL_GUARDED_BY(stream_mutex_);
   // The number of packets added to queue_.  Used to verify a packet at
   // Timestamp::PostStream() is the only Packet in the stream.
-  int64 num_packets_added_ GUARDED_BY(stream_mutex_);
-  Timestamp next_timestamp_bound_ GUARDED_BY(stream_mutex_);
+  int64 num_packets_added_ ABSL_GUARDED_BY(stream_mutex_);
+  Timestamp next_timestamp_bound_ ABSL_GUARDED_BY(stream_mutex_);
   // The |timestamp| argument passed to the last SelectAtTimestamp() call.
   // Ignored if enable_timestamps_ is false.
-  Timestamp last_select_timestamp_ GUARDED_BY(stream_mutex_);
-  bool closed_ GUARDED_BY(stream_mutex_);
+  Timestamp last_select_timestamp_ ABSL_GUARDED_BY(stream_mutex_);
+  bool closed_ ABSL_GUARDED_BY(stream_mutex_);
   // True if packet timestamps are used.
   bool enable_timestamps_ = true;
   std::string name_;
@@ -211,7 +211,7 @@ class InputStreamManager {
   Packet header_;
 
   // The maximum queue size for this stream if set.
-  int max_queue_size_ GUARDED_BY(stream_mutex_) = -1;
+  int max_queue_size_ ABSL_GUARDED_BY(stream_mutex_) = -1;
 
   // Callback to notify the framework that we have hit the maximum queue size.
   QueueSizeCallback becomes_full_callback_;

mediapipe/framework/output_stream_handler.h

@@ -92,7 +92,7 @@ class OutputStreamHandler {
   // resets data memebers.
   void PrepareForRun(
       const std::function<void(::mediapipe::Status)>& error_callback)
-      LOCKS_EXCLUDED(timestamp_mutex_);
+      ABSL_LOCKS_EXCLUDED(timestamp_mutex_);
 
   // Marks the output streams as started and propagates any changes made in
   // Calculator::Open().
@@ -106,10 +106,11 @@ class OutputStreamHandler {
   // Propagates timestamp directly if there is no ongoing parallel invocation.
   // Otherwise, updates task_timestamp_bound_.
   void UpdateTaskTimestampBound(Timestamp timestamp)
-      LOCKS_EXCLUDED(timestamp_mutex_);
+      ABSL_LOCKS_EXCLUDED(timestamp_mutex_);
 
   // Invoked after a call to Calculator::Process() function.
-  void PostProcess(Timestamp input_timestamp) LOCKS_EXCLUDED(timestamp_mutex_);
+  void PostProcess(Timestamp input_timestamp)
+      ABSL_LOCKS_EXCLUDED(timestamp_mutex_);
 
   // Propagates the output shards and closes all managed output streams.
   void Close(OutputStreamShardSet* output_shards);
@@ -133,7 +134,8 @@ class OutputStreamHandler {
                               OutputStreamShardSet* output_shards);
 
   // The packets and timestamp propagation logic for parallel execution.
-  virtual void PropagationLoop() EXCLUSIVE_LOCKS_REQUIRED(timestamp_mutex_) = 0;
+  virtual void PropagationLoop()
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(timestamp_mutex_) = 0;
 
   // Collection of all OutputStreamManager objects.
   OutputStreamManagerSet output_stream_managers_;
@@ -144,10 +146,11 @@ class OutputStreamHandler {
 
   absl::Mutex timestamp_mutex_;
   // A set of the completed input timestamps in ascending order.
-  std::set<Timestamp> completed_input_timestamps_ GUARDED_BY(timestamp_mutex_);
+  std::set<Timestamp> completed_input_timestamps_
+      ABSL_GUARDED_BY(timestamp_mutex_);
   // The current minimum timestamp for which a new packet could possibly arrive.
   // TODO: Rename the variable to be more descriptive.
-  Timestamp task_timestamp_bound_ GUARDED_BY(timestamp_mutex_);
+  Timestamp task_timestamp_bound_ ABSL_GUARDED_BY(timestamp_mutex_);
 
   // PropagateionState indicates the current state of the propagation process.
   // There are eight possible transitions:
@@ -187,7 +190,7 @@ class OutputStreamHandler {
     kPropagatingBound = 2,    //
     kPropagationPending = 3
   };
-  PropagationState propagation_state_ GUARDED_BY(timestamp_mutex_) = kIdle;
+  PropagationState propagation_state_ ABSL_GUARDED_BY(timestamp_mutex_) = kIdle;
 };
 
 using OutputStreamHandlerRegistry = GlobalFactoryRegistry<

mediapipe/framework/output_stream_manager.h

@@ -118,8 +118,8 @@ class OutputStreamManager {
   std::vector<Mirror> mirrors_;
 
   mutable absl::Mutex stream_mutex_;
-  Timestamp next_timestamp_bound_ GUARDED_BY(stream_mutex_);
-  bool closed_ GUARDED_BY(stream_mutex_);
+  Timestamp next_timestamp_bound_ ABSL_GUARDED_BY(stream_mutex_);
+  bool closed_ ABSL_GUARDED_BY(stream_mutex_);
 };
 
 }  // namespace mediapipe

mediapipe/framework/packet_generator_graph.cc

@@ -135,15 +135,15 @@ class GeneratorScheduler {
   void GenerateAndScheduleNext(int generator_index,
                                std::map<std::string, Packet>* side_packets,
                                std::unique_ptr<PacketSet> input_side_packet_set)
-      LOCKS_EXCLUDED(mutex_);
+      ABSL_LOCKS_EXCLUDED(mutex_);
 
   // Iterate through all generators in the config, scheduling any that
   // are runnable (and haven't been scheduled yet).
   void ScheduleAllRunnableGenerators(
-      std::map<std::string, Packet>* side_packets) LOCKS_EXCLUDED(mutex_);
+      std::map<std::string, Packet>* side_packets) ABSL_LOCKS_EXCLUDED(mutex_);
 
   // Waits until there are no pending tasks.
-  void WaitUntilIdle() LOCKS_EXCLUDED(mutex_);
+  void WaitUntilIdle() ABSL_LOCKS_EXCLUDED(mutex_);
 
   // Stores the indexes of the packet generators that were not scheduled (or
   // rather, not executed) in non_scheduled_generators. Returns the combined
@@ -158,26 +158,26 @@ class GeneratorScheduler {
 
   // Run all the application thread tasks (which are kept track of in
   // app_thread_tasks_).
-  void RunApplicationThreadTasks() LOCKS_EXCLUDED(app_thread_mutex_);
+  void RunApplicationThreadTasks() ABSL_LOCKS_EXCLUDED(app_thread_mutex_);
 
   const ValidatedGraphConfig* const validated_graph_;
   ::mediapipe::Executor* executor_;
 
   mutable absl::Mutex mutex_;
   // The number of pending tasks.
-  int num_tasks_ GUARDED_BY(mutex_) = 0;
+  int num_tasks_ ABSL_GUARDED_BY(mutex_) = 0;
   // This condition variable is signaled when num_tasks_ becomes 0.
   absl::CondVar idle_condvar_;
   // Accumulates the error statuses while running the packet generators.
-  std::vector<::mediapipe::Status> statuses_ GUARDED_BY(mutex_);
+  std::vector<::mediapipe::Status> statuses_ ABSL_GUARDED_BY(mutex_);
   // scheduled_generators_[i] is true if the packet generator with index i was
   // scheduled (or rather, executed).
-  std::vector<bool> scheduled_generators_ GUARDED_BY(mutex_);
+  std::vector<bool> scheduled_generators_ ABSL_GUARDED_BY(mutex_);
 
   absl::Mutex app_thread_mutex_;
   // Tasks to be executed on the application thread.
   std::deque<std::function<void()>> app_thread_tasks_
-      GUARDED_BY(app_thread_mutex_);
+      ABSL_GUARDED_BY(app_thread_mutex_);
   std::unique_ptr<internal::DelegatingExecutor> delegating_executor_;
 };
 

mediapipe/framework/profiler/BUILD

@@ -116,6 +116,7 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:name_util",
         "//mediapipe/framework/tool:tag_map",
         "//mediapipe/framework/tool:validate_name",
         "@com_google_absl//absl/memory",

mediapipe/framework/profiler/graph_profiler.cc

@@ -27,6 +27,7 @@
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/profiler/profiler_resource_util.h"
+#include "mediapipe/framework/tool/name_util.h"
 #include "mediapipe/framework/tool/tag_map.h"
 #include "mediapipe/framework/tool/validate_name.h"
 
@@ -133,7 +134,7 @@ void GraphProfiler::Initialize(
   for (int node_id = 0;
        node_id < validated_graph_config.CalculatorInfos().size(); ++node_id) {
     std::string node_name =
-        CanonicalNodeName(validated_graph_config.Config(), node_id);
+        tool::CanonicalNodeName(validated_graph_config.Config(), node_id);
     CalculatorProfile profile;
     profile.set_name(node_name);
     InitializeTimeHistogram(interval_size_usec, num_intervals,

mediapipe/framework/profiler/graph_profiler.h

@@ -122,15 +122,15 @@ class GraphProfiler : public std::enable_shared_from_this<ProfilingContext> {
   // the profiler disables itself and returns an empty stub if Initialize() is
   // called more than once.
   void Initialize(const ValidatedGraphConfig& validated_graph_config)
-      LOCKS_EXCLUDED(profiler_mutex_);
+      ABSL_LOCKS_EXCLUDED(profiler_mutex_);
 
   // Sets the profiler clock.
   void SetClock(const std::shared_ptr<mediapipe::Clock>& clock)
-      LOCKS_EXCLUDED(profiler_mutex_);
+      ABSL_LOCKS_EXCLUDED(profiler_mutex_);
 
   // Gets the profiler clock.
   const std::shared_ptr<mediapipe::Clock> GetClock() const
-      LOCKS_EXCLUDED(profiler_mutex_);
+      ABSL_LOCKS_EXCLUDED(profiler_mutex_);
 
   // Pauses profiling. No-op if already paused.
   void Pause();
@@ -138,7 +138,7 @@ class GraphProfiler : public std::enable_shared_from_this<ProfilingContext> {
   void Resume();
   // Resets cumulative profiling data. This only resets the information about
   // Process() and does NOT affect information for Open() and Close() methods.
-  void Reset() LOCKS_EXCLUDED(profiler_mutex_);
+  void Reset() ABSL_LOCKS_EXCLUDED(profiler_mutex_);
   // Begins profiling for a single graph run.
   ::mediapipe::Status Start(::mediapipe::Executor* executor);
   // Ends profiling for a single graph run.
@@ -150,8 +150,8 @@ class GraphProfiler : public std::enable_shared_from_this<ProfilingContext> {
   // Collects the runtime profile for Open(), Process(), and Close() of each
   // calculator in the graph. May be called at any time after the graph has been
   // initialized.
-  ::mediapipe::Status GetCalculatorProfiles(
-      std::vector<CalculatorProfile>*) const LOCKS_EXCLUDED(profiler_mutex_);
+  ::mediapipe::Status GetCalculatorProfiles(std::vector<CalculatorProfile>*)
+      const ABSL_LOCKS_EXCLUDED(profiler_mutex_);
 
   // Writes recent profiling and tracing data to a file specified in the
   // ProfilerConfig.  Includes events since the previous call to WriteProfile.
@@ -234,7 +234,7 @@ class GraphProfiler : public std::enable_shared_from_this<ProfilingContext> {
   // It is the responsibility of the caller to make sure the |timestamp_usec|
   // is valid for profiling.
   void AddPacketInfo(const TraceEvent& packet_info)
-      LOCKS_EXCLUDED(profiler_mutex_);
+      ABSL_LOCKS_EXCLUDED(profiler_mutex_);
   static void InitializeTimeHistogram(int64 interval_size_usec,
                                       int64 num_intervals,
                                       TimeHistogram* histogram);
@@ -273,10 +273,10 @@ class GraphProfiler : public std::enable_shared_from_this<ProfilingContext> {
 
   void SetOpenRuntime(const CalculatorContext& calculator_context,
                       int64 start_time_usec, int64 end_time_usec)
-      LOCKS_EXCLUDED(profiler_mutex_);
+      ABSL_LOCKS_EXCLUDED(profiler_mutex_);
   void SetCloseRuntime(const CalculatorContext& calculator_context,
                        int64 start_time_usec, int64 end_time_usec)
-      LOCKS_EXCLUDED(profiler_mutex_);
+      ABSL_LOCKS_EXCLUDED(profiler_mutex_);
 
   // Updates the input streams profiles for the calculator and returns the
   // minimum |source_process_start_usec| of all input packets, excluding empty
@@ -289,7 +289,7 @@ class GraphProfiler : public std::enable_shared_from_this<ProfilingContext> {
   // Requires ReaderLock for is_profiling_.
   void AddProcessSample(const CalculatorContext& calculator_context,
                         int64 start_time_usec, int64 end_time_usec)
-      LOCKS_EXCLUDED(profiler_mutex_);
+      ABSL_LOCKS_EXCLUDED(profiler_mutex_);
 
   // Helper method to get trace_log_path.  If the trace_log_path is empty and
   // tracing is enabled, this function returns a default platform dependent

mediapipe/framework/profiler/graph_tracer_test.cc

@@ -1284,5 +1284,33 @@ TEST_F(GraphTracerE2ETest, GpuTracing) {
   EXPECT_NE(nullptr, graph_.profiler()->CreateGlProfilingHelper());
 }
 
+// This test shows that ~CalculatorGraph() can complete successfully, even when
+// the periodic profiler output is enabled.  If periodic profiler output is not
+// stopped in ~CalculatorGraph(), it will deadlock at ~Executor().
+TEST_F(GraphTracerE2ETest, DestructGraph) {
+  std::string log_path = absl::StrCat(getenv("TEST_TMPDIR"), "/log_file_");
+  SetUpPassThroughGraph();
+  graph_config_.mutable_profiler_config()->set_trace_enabled(true);
+  graph_config_.mutable_profiler_config()->set_trace_log_path(log_path);
+  graph_config_.set_num_threads(4);
+
+  // Callbacks to control the LambdaCalculator.
+  ProcessFunction wait_0 = [&](const InputStreamShardSet& inputs,
+                               OutputStreamShardSet* outputs) {
+    return PassThrough(inputs, outputs);
+  };
+
+  {
+    CalculatorGraph graph;
+    // Start the graph with the callback.
+    MP_ASSERT_OK(graph.Initialize(graph_config_,
+                                  {
+                                      {"callback_0", Adopt(new auto(wait_0))},
+                                  }));
+    MP_ASSERT_OK(graph.StartRun({}));
+    // Destroy the graph immediately.
+  }
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/framework/profiler/sharded_map.h

@@ -47,7 +47,7 @@ class ShardedMap {
       : ShardedMap(capacity, capacity / 10 + 1) {}
 
   // Returns the iterator to the entry for a key.
-  inline iterator find(const Key& key) NO_THREAD_SAFETY_ANALYSIS {
+  inline iterator find(const Key& key) ABSL_NO_THREAD_SAFETY_ANALYSIS {
     size_t shard = Index(key);
     mutexes_[shard].Lock();
     typename Map::iterator iter = maps_[shard].find(key);
@@ -67,7 +67,7 @@ class ShardedMap {
 
   // Adds an entry to the map and returns the iterator to it.
   inline std::pair<iterator, bool> insert(const value_type& val)
-      NO_THREAD_SAFETY_ANALYSIS {
+      ABSL_NO_THREAD_SAFETY_ANALYSIS {
     size_t shard = Index(val.first);
     mutexes_[shard].Lock();
     std::pair<typename Map::iterator, bool> p = maps_[shard].insert(val);
@@ -91,7 +91,7 @@ class ShardedMap {
   inline size_t size() const { return size_; }
 
   // Returns the iterator to the first element.
-  inline iterator begin() NO_THREAD_SAFETY_ANALYSIS {
+  inline iterator begin() ABSL_NO_THREAD_SAFETY_ANALYSIS {
     mutexes_[0].Lock();
     iterator result{0, maps_[0].begin(), this};
     result.NextEntryShard();
@@ -153,14 +153,14 @@ class ShardedMap {
     Iterator(size_t shard, map_iterator iter, ShardedMapPtr map)
         : shard_(shard), iter_(iter), map_(map) {}
     // Releases all resources.
-    inline void Clear() NO_THREAD_SAFETY_ANALYSIS {
+    inline void Clear() ABSL_NO_THREAD_SAFETY_ANALYSIS {
       if (map_ && iter_ != map_->maps_.back().end()) {
         map_->mutexes_[shard_].Unlock();
       }
       map_ = nullptr;
     }
     // Moves to the shard of the next entry.
-    void NextEntryShard() NO_THREAD_SAFETY_ANALYSIS {
+    void NextEntryShard() ABSL_NO_THREAD_SAFETY_ANALYSIS {
       size_t last = map_->maps_.size() - 1;
       while (iter_ == map_->maps_[shard_].end() && shard_ < last) {
         map_->mutexes_[shard_].Unlock();

mediapipe/framework/scheduler.cc

@@ -238,7 +238,7 @@ void Scheduler::WaitUntilGraphInputStreamUnthrottled(
   }
   secondary_mutex->Unlock();
   ApplicationThreadAwait(
-      [this, seq_num]() EXCLUSIVE_LOCKS_REQUIRED(state_mutex_) {
+      [this, seq_num]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(state_mutex_) {
         return (unthrottle_seq_num_ != seq_num) || state_ == STATE_TERMINATED;
       });
   secondary_mutex->Lock();
@@ -255,7 +255,7 @@ void Scheduler::EmittedObservedOutput() {
 ::mediapipe::Status Scheduler::WaitForObservedOutput() {
   bool observed = false;
   ApplicationThreadAwait(
-      [this, &observed]() EXCLUSIVE_LOCKS_REQUIRED(state_mutex_) {
+      [this, &observed]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(state_mutex_) {
         observed = observed_output_signal_;
         observed_output_signal_ = false;
         waiting_for_observed_output_ = !observed && state_ != STATE_TERMINATED;
@@ -281,7 +281,7 @@ void Scheduler::EmittedObservedOutput() {
 
 ::mediapipe::Status Scheduler::WaitUntilDone() {
   RET_CHECK_NE(state_, STATE_NOT_STARTED);
-  ApplicationThreadAwait([this]() EXCLUSIVE_LOCKS_REQUIRED(state_mutex_) {
+  ApplicationThreadAwait([this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(state_mutex_) {
     return state_ == STATE_TERMINATED;
   });
   return ::mediapipe::OkStatus();

mediapipe/framework/scheduler.h

@@ -70,13 +70,13 @@ class Scheduler {
   // have been closed, and no more calculators can be run).
   // This function can be called only after Start().
   // Runs application thread tasks while waiting.
-  ::mediapipe::Status WaitUntilDone() LOCKS_EXCLUDED(state_mutex_);
+  ::mediapipe::Status WaitUntilDone() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Wait until the running graph is in the idle mode, which is when nothing can
   // be scheduled and nothing is running in the worker threads.  This function
   // can be called only after Start().
   // Runs application thread tasks while waiting.
-  ::mediapipe::Status WaitUntilIdle() LOCKS_EXCLUDED(state_mutex_);
+  ::mediapipe::Status WaitUntilIdle() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Wait until any graph input stream has been unthrottled.
   // This is meant to be used by CalculatorGraph::AddPacketToInputStream, which
@@ -86,14 +86,15 @@ class Scheduler {
   // This function can be called by multiple threads concurrently.
   // Runs application thread tasks while waiting.
   void WaitUntilGraphInputStreamUnthrottled(absl::Mutex* secondary_mutex)
-      LOCKS_EXCLUDED(state_mutex_) EXCLUSIVE_LOCKS_REQUIRED(secondary_mutex);
+      ABSL_LOCKS_EXCLUDED(state_mutex_)
+          ABSL_EXCLUSIVE_LOCKS_REQUIRED(secondary_mutex);
 
   // Wait until any observed output emits a packet. Like a semaphore,
   // this function returns immediately if an observed packet has already been
   // emitted since the previous call. This relies on the fact that the calls are
   // in sequence. Runs application thread tasks while waiting.
   // Returns ::mediapipe::OutOfRangeError if the graph terminated.
-  ::mediapipe::Status WaitForObservedOutput() LOCKS_EXCLUDED(state_mutex_);
+  ::mediapipe::Status WaitForObservedOutput() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Callback that is invoked by a node when it wants to be scheduled.
   // If the node is throttled, the call is ignored.
@@ -118,27 +119,28 @@ class Scheduler {
   void AddUnopenedSourceNode(CalculatorNode* node);
 
   // Adds |node| to |sources_queue_|.
-  void AddNodeToSourcesQueue(CalculatorNode* node) LOCKS_EXCLUDED(state_mutex_);
+  void AddNodeToSourcesQueue(CalculatorNode* node)
+      ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Assigns node to a scheduler queue.
   void AssignNodeToSchedulerQueue(CalculatorNode* node);
 
   // Pauses the scheduler.  Does nothing if Cancel has been called.
-  void Pause() LOCKS_EXCLUDED(state_mutex_);
+  void Pause() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Resumes the scheduler.
-  void Resume() LOCKS_EXCLUDED(state_mutex_);
+  void Resume() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Aborts the scheduler if the graph is started but is not terminated; no-op
   // otherwise.  For the graph to properly be cancelled, graph_->HasError()
   // must also return true.
-  void Cancel() LOCKS_EXCLUDED(state_mutex_);
+  void Cancel() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Returns true if scheduler is paused.
-  bool IsPaused() LOCKS_EXCLUDED(state_mutex_);
+  bool IsPaused() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Returns true if scheduler is terminated.
-  bool IsTerminated() LOCKS_EXCLUDED(state_mutex_);
+  bool IsTerminated() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Cleanup any remaining state after the run.
   void CleanupAfterRun();
@@ -148,11 +150,11 @@ class Scheduler {
   // Notifies the scheduler that a packet was added to a graph input stream.
   // The scheduler needs to check whether it is still deadlocked, and
   // unthrottle again if so.
-  void AddedPacketToGraphInputStream() LOCKS_EXCLUDED(state_mutex_);
+  void AddedPacketToGraphInputStream() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
-  void ThrottledGraphInputStream() LOCKS_EXCLUDED(state_mutex_);
-  void UnthrottledGraphInputStream() LOCKS_EXCLUDED(state_mutex_);
-  void EmittedObservedOutput() LOCKS_EXCLUDED(state_mutex_);
+  void ThrottledGraphInputStream() ABSL_LOCKS_EXCLUDED(state_mutex_);
+  void UnthrottledGraphInputStream() ABSL_LOCKS_EXCLUDED(state_mutex_);
+  void EmittedObservedOutput() ABSL_LOCKS_EXCLUDED(state_mutex_);
 
   // Closes all source nodes at the next scheduling opportunity.
   void CloseAllSourceNodes();
@@ -212,7 +214,7 @@ class Scheduler {
 
   // Returns true if nothing can be scheduled and no tasks are running or
   // scheduled to run on the Executor.
-  bool IsIdle() EXCLUSIVE_LOCKS_REQUIRED(state_mutex_);
+  bool IsIdle() ABSL_EXCLUSIVE_LOCKS_REQUIRED(state_mutex_);
 
   // Clean up active_sources_ by removing closed sources. If all the active
   // sources are closed, this will leave active_sources_ empty. If not, some
@@ -222,7 +224,8 @@ class Scheduler {
   // Adds the next layer of sources to the scheduler queue if the previous layer
   // has finished running.
   // Returns true if it scheduled any sources.
-  bool TryToScheduleNextSourceLayer() EXCLUSIVE_LOCKS_REQUIRED(state_mutex_);
+  bool TryToScheduleNextSourceLayer()
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(state_mutex_);
 
   // Takes care of three different operations, as needed:
   // - activating sources;
@@ -230,10 +233,10 @@ class Scheduler {
   // - terminating the scheduler.
   // Thread-safe and reentrant.
   // TODO: analyze call sites, split it up further.
-  void HandleIdle() EXCLUSIVE_LOCKS_REQUIRED(state_mutex_);
+  void HandleIdle() ABSL_EXCLUSIVE_LOCKS_REQUIRED(state_mutex_);
 
   // Terminates the scheduler. Should only be called by HandleIdle.
-  void Quit() EXCLUSIVE_LOCKS_REQUIRED(state_mutex_);
+  void Quit() ABSL_EXCLUSIVE_LOCKS_REQUIRED(state_mutex_);
 
   // Helper for the various Wait methods. Waits for the given condition,
   // running application thread tasks in the meantime.
@@ -257,7 +260,7 @@ class Scheduler {
   // Priority queue of source nodes ordered by layer and then source process
   // order. This stores the set of sources that are yet to be run.
   std::priority_queue<SchedulerQueue::Item> sources_queue_
-      GUARDED_BY(state_mutex_);
+      ABSL_GUARDED_BY(state_mutex_);
 
   // Source nodes with the smallest source layer are at the beginning of
   // unopened_sources_. Before the scheduler is started, all source nodes are
@@ -276,7 +279,7 @@ class Scheduler {
   // These correspond to the Wait* methods in this class.
   // Not all state changes need to signal this, only those that enter one of
   // the waitable states.
-  absl::CondVar state_cond_var_ GUARDED_BY(state_mutex_);
+  absl::CondVar state_cond_var_ ABSL_GUARDED_BY(state_mutex_);
 
   // Number of queues which are not idle.
   // Note: this indicates two slightly different things:
@@ -288,17 +291,18 @@ class Scheduler {
   // This is ok, because it happens within a single critical section, which is
   // guarded by state_mutex_. If we wanted to split this critical section, we
   // would have to separate a and b into two variables.
-  int non_idle_queue_count_ GUARDED_BY(state_mutex_) = 0;
+  int non_idle_queue_count_ ABSL_GUARDED_BY(state_mutex_) = 0;
 
   // Tasks to be executed on the application thread.
-  std::deque<std::function<void()>> app_thread_tasks_ GUARDED_BY(state_mutex_);
+  std::deque<std::function<void()>> app_thread_tasks_
+      ABSL_GUARDED_BY(state_mutex_);
 
   // Used by HandleIdle to avoid multiple concurrent executions.
   // We cannot simply hold a mutex throughout it, for two reasons:
   // - We need it to be reentrant, which Mutex does not support.
   // - We want simultaneous calls to return immediately instead of waiting,
   //   and Mutex's TryLock is not guaranteed to work.
-  bool handling_idle_ GUARDED_BY(state_mutex_) = false;
+  bool handling_idle_ ABSL_GUARDED_BY(state_mutex_) = false;
 
   // Mutex for the scheduler state and related things.
   // Note: state_ is declared as atomic so that its getter methods don't need
@@ -309,19 +313,19 @@ class Scheduler {
   std::atomic<State> state_ = ATOMIC_VAR_INIT(STATE_NOT_STARTED);
 
   // True if all graph input streams are closed.
-  bool graph_input_streams_closed_ GUARDED_BY(state_mutex_) = false;
+  bool graph_input_streams_closed_ ABSL_GUARDED_BY(state_mutex_) = false;
 
   // Number of throttled graph input streams.
-  int throttled_graph_input_stream_count_ GUARDED_BY(state_mutex_) = 0;
+  int throttled_graph_input_stream_count_ ABSL_GUARDED_BY(state_mutex_) = 0;
 
   // Used to stop WaitUntilGraphInputStreamUnthrottled.
-  int unthrottle_seq_num_ GUARDED_BY(state_mutex_) = 0;
+  int unthrottle_seq_num_ ABSL_GUARDED_BY(state_mutex_) = 0;
 
   // Used to stop WaitForObservedOutput.
-  bool observed_output_signal_ GUARDED_BY(state_mutex_) = false;
+  bool observed_output_signal_ ABSL_GUARDED_BY(state_mutex_) = false;
 
   // True if an application thread is waiting in WaitForObservedOutput.
-  bool waiting_for_observed_output_ GUARDED_BY(state_mutex_) = false;
+  bool waiting_for_observed_output_ ABSL_GUARDED_BY(state_mutex_) = false;
 };
 
 }  // namespace internal

mediapipe/framework/scheduler_queue.h

@@ -105,45 +105,45 @@ class SchedulerQueue : public TaskQueue {
   // NOTE: After calling SetRunning(true), the caller must call
   // SubmitWaitingTasksToExecutor since tasks may have been added while the
   // queue was not running.
-  void SetRunning(bool running) LOCKS_EXCLUDED(mutex_);
+  void SetRunning(bool running) ABSL_LOCKS_EXCLUDED(mutex_);
 
   // Gets the number of tasks that need to be submitted to the executor, and
   // updates num_pending_tasks_. If this method is called and returns a
   // non-zero value, the executor's AddTask method *must* be called for each
   // task returned, but it can be called without holding the lock.
-  int GetTasksToSubmitToExecutor() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+  int GetTasksToSubmitToExecutor() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
 
   // Submits tasks that are waiting (e.g. that were added while the queue was
   // not running) if the queue is running. The caller must not hold any mutex.
-  void SubmitWaitingTasksToExecutor() LOCKS_EXCLUDED(mutex_);
+  void SubmitWaitingTasksToExecutor() ABSL_LOCKS_EXCLUDED(mutex_);
 
   // Adds a node and a calculator context to the scheduler queue if the node is
   // not already running. Note that if the node was running, then it will be
   // rescheduled upon completion (after checking dependencies), so this call is
   // not lost.
   void AddNode(CalculatorNode* node, CalculatorContext* cc)
-      LOCKS_EXCLUDED(mutex_);
+      ABSL_LOCKS_EXCLUDED(mutex_);
 
   // Adds a node to the scheduler queue for an OpenNode() call.
-  void AddNodeForOpen(CalculatorNode* node) LOCKS_EXCLUDED(mutex_);
+  void AddNodeForOpen(CalculatorNode* node) ABSL_LOCKS_EXCLUDED(mutex_);
 
   // Adds an Item to queue_.
   void AddItemToQueue(Item&& item);
 
-  void CleanupAfterRun() LOCKS_EXCLUDED(mutex_);
+  void CleanupAfterRun() ABSL_LOCKS_EXCLUDED(mutex_);
 
  private:
   // Used internally by RunNextTask. Invokes ProcessNode or CloseNode, followed
   // by EndScheduling.
   void RunCalculatorNode(CalculatorNode* node, CalculatorContext* cc)
-      LOCKS_EXCLUDED(mutex_);
+      ABSL_LOCKS_EXCLUDED(mutex_);
 
   // Used internally by RunNextTask. Invokes OpenNode, followed by
   // CheckIfBecameReady.
-  void OpenCalculatorNode(CalculatorNode* node) LOCKS_EXCLUDED(mutex_);
+  void OpenCalculatorNode(CalculatorNode* node) ABSL_LOCKS_EXCLUDED(mutex_);
 
   // Checks whether the queue has no queued nodes or pending tasks.
-  bool IsIdle() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+  bool IsIdle() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
 
   Executor* executor_ = nullptr;
 
@@ -154,16 +154,16 @@ class SchedulerQueue : public TaskQueue {
   // decrements it. The queue is running if running_count_ > 0. A running
   // queue will submit tasks to the executor.
   // Invariant: running_count_ <= 1.
-  int running_count_ GUARDED_BY(mutex_) = 0;
+  int running_count_ ABSL_GUARDED_BY(mutex_) = 0;
 
   // Number of tasks added to the Executor and not yet complete.
-  int num_pending_tasks_ GUARDED_BY(mutex_);
+  int num_pending_tasks_ ABSL_GUARDED_BY(mutex_);
 
   // Number of tasks that need to be added to the Executor.
-  int num_tasks_to_add_ GUARDED_BY(mutex_);
+  int num_tasks_to_add_ ABSL_GUARDED_BY(mutex_);
 
   // Queue of nodes that need to be run.
-  std::priority_queue<Item> queue_ GUARDED_BY(mutex_);
+  std::priority_queue<Item> queue_ ABSL_GUARDED_BY(mutex_);
 
   SchedulerShared* const shared_;
 

mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc

@@ -67,7 +67,7 @@ class FixedSizeInputStreamHandler : public DefaultInputStreamHandler {
 
  private:
   // Drops packets if all input streams exceed trigger_queue_size.
-  void EraseAllSurplus() EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_) {
+  void EraseAllSurplus() ABSL_EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_) {
     Timestamp min_timestamp_all_streams = Timestamp::Max();
     for (const auto& stream : input_stream_managers_) {
       // Check whether every InputStreamImpl grew beyond trigger_queue_size.
@@ -127,7 +127,8 @@ class FixedSizeInputStreamHandler : public DefaultInputStreamHandler {
   // Keeps only the most recent target_queue_size packets in each stream
   // exceeding trigger_queue_size.  Also, discards all packets older than the
   // first kept timestamp on any stream.
-  void EraseAnySurplus(bool keep_one) EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_) {
+  void EraseAnySurplus(bool keep_one)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_) {
     // Record the most recent first kept timestamp on any stream.
     for (const auto& stream : input_stream_managers_) {
       int32 queue_size = (stream->QueueSize() >= trigger_queue_size_)
@@ -151,7 +152,7 @@ class FixedSizeInputStreamHandler : public DefaultInputStreamHandler {
   }
 
   void EraseSurplusPackets(bool keep_one)
-      EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_) {
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_) {
     return (fixed_min_size_) ? EraseAllSurplus() : EraseAnySurplus(keep_one);
   }
 
@@ -218,9 +219,9 @@ class FixedSizeInputStreamHandler : public DefaultInputStreamHandler {
   bool fixed_min_size_;
   // Indicates that GetNodeReadiness has returned kReadyForProcess once, and
   // the corresponding call to FillInputSet has not yet completed.
-  bool pending_ GUARDED_BY(erase_mutex_);
+  bool pending_ ABSL_GUARDED_BY(erase_mutex_);
   // The timestamp used to truncate all input streams.
-  Timestamp kept_timestamp_ GUARDED_BY(erase_mutex_);
+  Timestamp kept_timestamp_ ABSL_GUARDED_BY(erase_mutex_);
   absl::Mutex erase_mutex_;
 };
 

mediapipe/framework/stream_handler/fixed_size_input_stream_handler_test.cc

@@ -35,13 +35,13 @@ const int64 kSlowCalculatorRate = 10;
 
 // Rate limiter for TestSlowCalculator.
 ABSL_CONST_INIT absl::Mutex g_source_mutex(absl::kConstInit);
-int64 g_source_counter GUARDED_BY(g_source_mutex);
+int64 g_source_counter ABSL_GUARDED_BY(g_source_mutex);
 
 // Rate limiter for TestSourceCalculator.
-int64 g_slow_counter GUARDED_BY(g_source_mutex);
+int64 g_slow_counter ABSL_GUARDED_BY(g_source_mutex);
 
 // Flag that indicates that the source is done.
-bool g_source_done GUARDED_BY(g_source_mutex);
+bool g_source_done ABSL_GUARDED_BY(g_source_mutex);
 
 class TestSourceCalculator : public CalculatorBase {
  public:
@@ -74,7 +74,7 @@ class TestSourceCalculator : public CalculatorBase {
   }
 
  private:
-  bool CanProceed() const EXCLUSIVE_LOCKS_REQUIRED(g_source_mutex) {
+  bool CanProceed() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(g_source_mutex) {
     return g_source_counter <= kSlowCalculatorRate * g_slow_counter ||
            g_source_counter <= 1;
   }
@@ -109,7 +109,7 @@ class TestSlowCalculator : public CalculatorBase {
   }
 
  private:
-  bool CanProceed() const EXCLUSIVE_LOCKS_REQUIRED(g_source_mutex) {
+  bool CanProceed() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(g_source_mutex) {
     return g_source_counter > kSlowCalculatorRate * g_slow_counter ||
            g_source_done;
   }

mediapipe/framework/stream_handler/in_order_output_stream_handler.h

@@ -40,15 +40,15 @@ class InOrderOutputStreamHandler : public OutputStreamHandler {
                             options, calculator_run_in_parallel) {}
 
  private:
-  void PropagationLoop() EXCLUSIVE_LOCKS_REQUIRED(timestamp_mutex_) final;
+  void PropagationLoop() ABSL_EXCLUSIVE_LOCKS_REQUIRED(timestamp_mutex_) final;
 
   void PropagatePackets(CalculatorContext** calculator_context,
                         Timestamp* context_timestamp)
-      EXCLUSIVE_LOCKS_REQUIRED(timestamp_mutex_);
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(timestamp_mutex_);
 
   void PropagationBound(CalculatorContext** calculator_context,
                         Timestamp* context_timestamp)
-      EXCLUSIVE_LOCKS_REQUIRED(timestamp_mutex_);
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(timestamp_mutex_);
 };
 }  // namespace mediapipe
 

mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc

@@ -64,18 +64,18 @@ class SyncSetInputStreamHandler : public InputStreamHandler {
   // Populates timestamp bounds for streams outside the ready sync-set.
   void FillInputBounds(Timestamp input_timestamp,
                        InputStreamShardSet* input_set)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
 
  private:
   absl::Mutex mutex_;
   // The ids of each set of inputs.
-  std::vector<std::vector<CollectionItemId>> sync_sets_ GUARDED_BY(mutex_);
+  std::vector<std::vector<CollectionItemId>> sync_sets_ ABSL_GUARDED_BY(mutex_);
   // The index of the ready sync set.  A value of -1 indicates that no
   // sync sets are ready.
-  int ready_sync_set_index_ GUARDED_BY(mutex_) = -1;
+  int ready_sync_set_index_ ABSL_GUARDED_BY(mutex_) = -1;
   // The timestamp at which the sync set is ready.  If no sync set is
   // ready then this variable should be Timestamp::Done() .
-  Timestamp ready_timestamp_ GUARDED_BY(mutex_);
+  Timestamp ready_timestamp_ ABSL_GUARDED_BY(mutex_);
 };
 
 REGISTER_INPUT_STREAM_HANDLER(SyncSetInputStreamHandler);