Project import generated by Copybara.

GitOrigin-RevId: 5aca6b3f07b67e09988a901f50f595ca5f566e67

mediapipe/framework/BUILD

@@ -14,13 +14,12 @@
 # limitations under the License.
 #
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_py_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_py_proto_library")
-
 package_group(
     name = "mediapipe_internal",
     packages = [
@@ -464,6 +463,8 @@ cc_library(
         "//mediapipe/framework:packet_generator_cc_proto",
         "//mediapipe/framework:status_handler_cc_proto",
         "//mediapipe/framework:thread_pool_executor_cc_proto",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//mediapipe/gpu:graph_support",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/container:fixed_array",
@@ -1272,6 +1273,7 @@ cc_library(
         "//mediapipe/framework/tool:validate",
         "//mediapipe/framework/tool:validate_name",
         "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],

mediapipe/framework/calculator_graph.cc

@@ -24,6 +24,7 @@
 #include <vector>
 
 #include "absl/container/fixed_array.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
@@ -1017,8 +1018,8 @@ void CalculatorGraph::UpdateThrottledNodes(InputStreamManager* stream,
   // TODO Change the throttling code to use the index directly
   // rather than looking up a stream name.
   int node_index = validated_graph_->OutputStreamToNode(stream->Name());
-  std::unordered_set<int> owned_set;
-  const std::unordered_set<int>* upstream_nodes;
+  absl::flat_hash_set<int> owned_set;
+  const absl::flat_hash_set<int>* upstream_nodes;
   if (node_index >= validated_graph_->CalculatorInfos().size()) {
     // TODO just create a NodeTypeInfo object for each virtual node.
     owned_set.insert(node_index);
@@ -1100,10 +1101,10 @@ bool CalculatorGraph::UnthrottleSources() {
   // This is a sufficient because succesfully growing at least one full input
   // stream during each call to UnthrottleSources will eventually resolve
   // each deadlock.
-  std::unordered_set<InputStreamManager*> full_streams;
+  absl::flat_hash_set<InputStreamManager*> full_streams;
   {
     absl::MutexLock lock(&full_input_streams_mutex_);
-    for (std::unordered_set<InputStreamManager*>& s : full_input_streams_) {
+    for (absl::flat_hash_set<InputStreamManager*>& s : full_input_streams_) {
       if (!s.empty()) {
         full_streams.insert(s.begin(), s.end());
       }

mediapipe/framework/calculator_graph.h

@@ -23,13 +23,13 @@
 #include <map>
 #include <memory>
 #include <string>
-#include <unordered_map>
-#include <unordered_set>
 #include <utility>
 #include <vector>
 
 #include "absl/base/macros.h"
 #include "absl/container/fixed_array.h"
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/synchronization/mutex.h"
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_base.h"
@@ -579,18 +579,18 @@ class CalculatorGraph {
   // A node is scheduled only if this set is empty.  Similarly, a packet
   // is added to a graph input stream only if this set is empty.
   // Note that this vector contains an unused entry for each non-source node.
-  std::vector<std::unordered_set<InputStreamManager*>> full_input_streams_
+  std::vector<absl::flat_hash_set<InputStreamManager*>> full_input_streams_
       GUARDED_BY(full_input_streams_mutex_);
 
   // Maps stream names to graph input stream objects.
-  std::unordered_map<std::string, std::unique_ptr<GraphInputStream>>
+  absl::flat_hash_map<std::string, std::unique_ptr<GraphInputStream>>
       graph_input_streams_;
 
   // Maps graph input streams to their virtual node ids.
-  std::unordered_map<std::string, int> graph_input_stream_node_ids_;
+  absl::flat_hash_map<std::string, int> graph_input_stream_node_ids_;
 
   // Maps graph input streams to their max queue size.
-  std::unordered_map<std::string, int> graph_input_stream_max_queue_size_;
+  absl::flat_hash_map<std::string, int> graph_input_stream_max_queue_size_;
 
   // The factory for making counters associated with this graph.
   std::unique_ptr<CounterFactory> counter_factory_;

mediapipe/framework/calculator_graph_side_packet_test.cc

@@ -68,6 +68,7 @@ class CountAndOutputSummarySidePacketInCloseCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Close(CalculatorContext* cc) final {
+    absl::SleepFor(absl::Milliseconds(300));  // For GetOutputSidePacket test.
     cc->OutputSidePackets().Index(0).Set(
         MakePacket<int>(count_).At(Timestamp::Unset()));
     return ::mediapipe::OkStatus();

mediapipe/framework/graph_validation_test.cc

@@ -32,7 +32,7 @@ namespace mediapipe {
 namespace {
 
 // Shows validation success for a graph and a subgraph.
-TEST(ValidatedGraphConfigTest, InitializeGraphFromProtos) {
+TEST(GraphValidationTest, InitializeGraphFromProtos) {
   auto config_1 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
     type: "PassThroughGraph"
     input_stream: "INPUT:stream_1"
@@ -102,7 +102,7 @@ TEST(ValidatedGraphConfigTest, InitializeGraphFromProtos) {
 }
 
 // Shows validation failure due to an unregistered subgraph.
-TEST(ValidatedGraphConfigTest, InitializeGraphFromLinker) {
+TEST(GraphValidationTest, InitializeGraphFromLinker) {
   EXPECT_FALSE(SubgraphRegistry::IsRegistered("DubQuadTestSubgraph"));
   ValidatedGraphConfig builder_1;
   ::mediapipe::Status status_1 =
@@ -114,7 +114,7 @@ TEST(ValidatedGraphConfigTest, InitializeGraphFromLinker) {
 }
 
 // Shows validation success for a graph and a template subgraph.
-TEST(ValidatedGraphConfigTest, InitializeTemplateFromProtos) {
+TEST(GraphValidationTest, InitializeTemplateFromProtos) {
   mediapipe::tool::TemplateParser::Parser parser;
   CalculatorGraphTemplate config_1;
   CHECK(parser.ParseFromString(R"(
@@ -210,5 +210,109 @@ TEST(ValidatedGraphConfigTest, InitializeTemplateFromProtos) {
       )")));
 }
 
+// Shows passing validation of optional subgraph inputs and output streams.
+TEST(GraphValidationTest, OptionalSubgraphStreams) {
+  // A subgraph defining two optional input streams
+  // and two optional output streams.
+  auto config_1 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    type: "PassThroughGraph"
+    input_stream: "INPUT:input_0"
+    input_stream: "INPUT:1:input_1"
+    output_stream: "OUTPUT:output_0"
+    output_stream: "OUTPUT:1:output_1"
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "input_0"    # Any Type.
+      input_stream: "input_1"    # Any Type.
+      output_stream: "output_0"  # Same as input.
+    }
+  )");
+
+  // An enclosing graph that specifies one of the two optional input streams
+  // and one of the two optional output streams.
+  auto config_2 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "INPUT:foo_in"
+    output_stream: "OUTPUT:foo_out"
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "foo_in"    # Any Type.
+      output_stream: "foo_bar"  # Same as input.
+    }
+    node {
+      calculator: "PassThroughGraph"
+      input_stream: "INPUT:foo_bar"    # Any Type.
+      output_stream: "OUTPUT:foo_out"  # Same as input.
+    }
+  )");
+
+  GraphValidation validation_1;
+  MP_EXPECT_OK(validation_1.Validate({config_1, config_2}, {}));
+  CalculatorGraph graph_1;
+  MP_EXPECT_OK(graph_1.Initialize({config_1, config_2}, {}));
+  EXPECT_THAT(
+      graph_1.Config(),
+
+      // The result includes only the requested input and output streams.
+      EqualsProto(::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "INPUT:foo_in"
+        output_stream: "OUTPUT:foo_out"
+        node {
+          calculator: "PassThroughCalculator"
+          input_stream: "foo_in"
+          output_stream: "foo_bar"
+        }
+        node {
+          calculator: "PassThroughCalculator"
+          input_stream: "foo_bar"
+          output_stream: "foo_out"
+        }
+        executor {}
+      )")));
+}
+
+// Shows failing validation of optional subgraph inputs and output streams.
+TEST(GraphValidationTest, OptionalSubgraphStreamsMismatched) {
+  // A subgraph defining two optional input streams
+  // and two optional output streams.
+  auto config_1 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    type: "PassThroughGraph"
+    input_stream: "INPUT:input_0"
+    input_stream: "INPUT:1:input_1"
+    output_stream: "OUTPUT:output_0"
+    output_stream: "OUTPUT:1:output_1"
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "input_0"    # Any Type.
+      input_stream: "input_1"    # Any Type.
+      output_stream: "output_0"  # Same as input.
+    }
+  )");
+
+  // An enclosing graph that specifies one of the two optional input streams
+  // and both of the two optional output streams.
+  auto config_2 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "INPUT:foo_in"
+    output_stream: "OUTPUT:foo_out"
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "foo_in"    # Any Type.
+      output_stream: "foo_bar"  # Same as input.
+    }
+    node {
+      calculator: "PassThroughGraph"
+      input_stream: "INPUT:foo_bar"    # Any Type.
+      input_stream: "INPUT:1:foo_bar"  # Any Type.
+      output_stream: "OUTPUT:foo_out"  # Same as input.
+    }
+  )");
+
+  GraphValidation validation_1;
+  mediapipe::Status status = validation_1.Validate({config_1, config_2}, {});
+  ASSERT_EQ(status.code(), ::mediapipe::StatusCode::kInvalidArgument);
+  ASSERT_THAT(status.ToString(),
+              testing::HasSubstr(
+                  "PassThroughCalculator must use matching tags and indexes"));
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/framework/packet.h

@@ -85,7 +85,7 @@ class Packet {
   // given timestamp. Does not modify *this.
   Packet At(class Timestamp timestamp) const&;
 
-  // The rvalue reference overload of Packet's memeber function
+  // The rvalue reference overload of Packet's member function
   // Packet::At(class Timestamp). Moves *this to a new Packet and returns
   // the new Packet with the given timestamp.
   Packet At(class Timestamp timestamp) &&;

mediapipe/framework/port/BUILD

@@ -247,6 +247,26 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "opencv_features2d",
+    hdrs = ["opencv_features2d_inc.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":opencv_core",
+        "//third_party:opencv",
+    ],
+)
+
+cc_library(
+    name = "opencv_calib3d",
+    hdrs = ["opencv_calib3d_inc.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":opencv_core",
+        "//third_party:opencv",
+    ],
+)
+
 cc_library(
     name = "parse_text_proto",
     hdrs = [

mediapipe/framework/port/opencv_calib3d_inc.h

@@ -0,0 +1,26 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FRAMEWORK_PORT_OPENCV_CALIB3D_INC_H_
+#define MEDIAPIPE_FRAMEWORK_PORT_OPENCV_CALIB3D_INC_H_
+
+#include <opencv2/core/version.hpp>
+
+#ifdef CV_VERSION_EPOCH  // for OpenCV 2.x
+#include <opencv2/calib3d/calib3d.hpp>
+#else
+#include <opencv2/calib3d.hpp>
+#endif
+
+#endif  // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_CALIB3D_INC_H_

mediapipe/framework/port/opencv_features2d_inc.h

@@ -0,0 +1,26 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_
+#define MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_
+
+#include <opencv2/core/version.hpp>
+
+#ifdef CV_VERSION_EPOCH  // for OpenCV 2.x
+#include <opencv2/features2d/features2d.hpp>
+#else
+#include <opencv2/features2d.hpp>
+#endif
+
+#endif  // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_

mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc

@@ -155,7 +155,7 @@ class FixedSizeInputStreamHandler : public DefaultInputStreamHandler {
     return (fixed_min_size_) ? EraseAllSurplus() : EraseAnySurplus(keep_one);
   }
 
-  NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) {
+  NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override {
     DCHECK(min_stream_timestamp);
     absl::MutexLock lock(&erase_mutex_);
     // kReadyForProcess is returned only once until FillInputSet completes.

mediapipe/framework/testdata/BUILD

@@ -31,7 +31,7 @@ mediapipe_cc_proto_library(
     name = "sky_light_calculator_cc_proto",
     srcs = ["sky_light_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":sky_light_calculator_proto"],
 )
 
@@ -45,7 +45,7 @@ mediapipe_cc_proto_library(
     name = "night_light_calculator_cc_proto",
     srcs = ["night_light_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":night_light_calculator_proto"],
 )
 

mediapipe/framework/tool/BUILD

@@ -177,7 +177,7 @@ cc_library(
     deps = [
         "//mediapipe/framework:packet",
         "//mediapipe/framework/port:statusor",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 

mediapipe/framework/tool/mediapipe_graph.bzl

@@ -110,7 +110,7 @@ def mediapipe_simple_subgraph(
       testonly: pass 1 if the graph is to be used only for tests.
       **kwargs: Remaining keyword args, forwarded to cc_library.
     """
-    graph_base_name = graph.replace(":", "/").split("/")[-1].rsplit(".", 1)[0]
+    graph_base_name = name
     mediapipe_binary_graph(
         name = name + "_graph",
         graph = graph,

mediapipe/framework/tool/subgraph_expansion.cc

@@ -52,6 +52,31 @@ namespace tool {
   return ::mediapipe::OkStatus();
 }
 
+// Returns subgraph streams not requested by a subgraph-node.
+::mediapipe::Status FindIgnoredStreams(
+    const proto_ns::RepeatedPtrField<ProtoString>& src_streams,
+    const proto_ns::RepeatedPtrField<ProtoString>& dst_streams,
+    std::set<std::string>* result) {
+  ASSIGN_OR_RETURN(auto src_map, tool::TagMap::Create(src_streams));
+  ASSIGN_OR_RETURN(auto dst_map, tool::TagMap::Create(dst_streams));
+  std::set_difference(src_map->Names().begin(), src_map->Names().end(),
+                      dst_map->Names().begin(), dst_map->Names().end(),
+                      std::inserter(*result, result->begin()));
+  return ::mediapipe::OkStatus();
+}
+
+// Removes subgraph streams not requested by a subgraph-node.
+::mediapipe::Status RemoveIgnoredStreams(
+    proto_ns::RepeatedPtrField<ProtoString>* streams,
+    const std::set<std::string>& missing_streams) {
+  for (int i = streams->size() - 1; i >= 0; --i) {
+    if (missing_streams.count(streams->Get(i)) > 0) {
+      streams->DeleteSubrange(i, 1);
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status TransformNames(
     CalculatorGraphConfig* config,
     const std::function<std::string(absl::string_view)>& transform) {
@@ -190,6 +215,14 @@ static ::mediapipe::Status PrefixNames(int subgraph_index,
           .SetPrepend()
       << "while processing the output side packets of subgraph node "
       << subgraph_node.calculator() << ": ";
+  std::set<std::string> ignored_input_streams;
+  MP_RETURN_IF_ERROR(FindIgnoredStreams(subgraph_config->input_stream(),
+                                        subgraph_node.input_stream(),
+                                        &ignored_input_streams));
+  std::set<std::string> ignored_input_side_packets;
+  MP_RETURN_IF_ERROR(FindIgnoredStreams(subgraph_config->input_side_packet(),
+                                        subgraph_node.input_side_packet(),
+                                        &ignored_input_side_packets));
   std::map<std::string, std::string>* name_map;
   auto replace_names = [&name_map](absl::string_view s) {
     std::string original(s);
@@ -207,6 +240,12 @@ static ::mediapipe::Status PrefixNames(int subgraph_index,
         TransformStreamNames(node.mutable_input_side_packet(), replace_names));
     MP_RETURN_IF_ERROR(
         TransformStreamNames(node.mutable_output_side_packet(), replace_names));
+
+    // Remove input streams and side packets ignored by the subgraph-node.
+    MP_RETURN_IF_ERROR(RemoveIgnoredStreams(node.mutable_input_stream(),
+                                            ignored_input_streams));
+    MP_RETURN_IF_ERROR(RemoveIgnoredStreams(node.mutable_input_side_packet(),
+                                            ignored_input_side_packets));
   }
   name_map = &side_packet_map;
   for (auto& generator : *subgraph_config->mutable_packet_generator()) {

mediapipe/framework/validated_graph_config.cc

@@ -14,8 +14,7 @@
 
 #include "mediapipe/framework/validated_graph_config.h"
 
-#include <unordered_set>
-
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
@@ -934,7 +933,7 @@ NodeTypeInfo::NodeRef ValidatedGraphConfig::NodeForSorterIndex(
 }
 
 ::mediapipe::Status ValidatedGraphConfig::ValidateExecutors() {
-  std::unordered_set<ProtoString> declared_names;
+  absl::flat_hash_set<ProtoString> declared_names;
   for (const ExecutorConfig& executor_config : config_.executor()) {
     if (IsReservedExecutorName(executor_config.name())) {
       return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
@@ -964,7 +963,7 @@ NodeTypeInfo::NodeRef ValidatedGraphConfig::NodeForSorterIndex(
              << "\"" << executor_name << "\" is a reserved executor name.";
     }
     // The executor must be declared in an ExecutorConfig.
-    if (declared_names.find(executor_name) == declared_names.end()) {
+    if (!declared_names.contains(executor_name)) {
       return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
              << "The executor \"" << executor_name
              << "\" is not declared in an ExecutorConfig.";

mediapipe/framework/validated_graph_config.h

@@ -16,9 +16,9 @@
 #define MEDIAPIPE_FRAMEWORK_VALIDATED_GRAPH_CONFIG_H_
 
 #include <map>
-#include <unordered_set>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_contract.h"
 #include "mediapipe/framework/packet_generator.pb.h"
@@ -169,7 +169,7 @@ class NodeTypeInfo {
   // be a virtual node corresponding to a graph input stream (which are
   // listed by index contiguously after all calculators).
   // This function is only valid for a NodeTypeInfo of NodeType CALCULATOR.
-  const std::unordered_set<int>& AncestorSources() const {
+  const absl::flat_hash_set<int>& AncestorSources() const {
     return ancestor_sources_;
   }
   // Returns True if the source was not already there.
@@ -213,7 +213,7 @@ class NodeTypeInfo {
   NodeRef node_;
 
   // The set of sources which affect this node.
-  std::unordered_set<int> ancestor_sources_;
+  absl::flat_hash_set<int> ancestor_sources_;
 };
 
 // Information for either the input or output side of an edge.  An edge