Project import generated by Copybara.

GitOrigin-RevId: f72a0f86c2c2acdb1920973c718a9e26ed3ec4b6

mediapipe/framework/calculator_graph.cc

@@ -760,10 +760,6 @@ CalculatorGraph::PrepareGpu(const std::map<std::string, Packet>& side_packets) {
 }
 
 ::mediapipe::Status CalculatorGraph::WaitUntilIdle() {
-  if (has_sources_) {
-    return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
-           << "WaitUntilIdle called on a graph with source nodes.";
-  }
   MP_RETURN_IF_ERROR(scheduler_.WaitUntilIdle());
   VLOG(2) << "Scheduler idle.";
   ::mediapipe::Status status = ::mediapipe::OkStatus();

mediapipe/framework/calculator_graph_test.cc

@@ -65,6 +65,9 @@ namespace mediapipe {
 
 namespace {
 
+using testing::ElementsAre;
+using testing::HasSubstr;
+
 // Pass packets through. Note that it calls SetOffset() in Process()
 // instead of Open().
 class SetOffsetInProcessCalculator : public CalculatorBase {

mediapipe/framework/deps/BUILD

@@ -272,6 +272,14 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "re2",
+    hdrs = [
+        "re2.h",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 cc_library(
     name = "status_matchers",
     testonly = 1,
@@ -280,7 +288,7 @@ cc_library(
     visibility = ["//mediapipe/framework/port:__pkg__"],
     deps = [
         ":status",
-        "@com_google_googletest//:gtest_main",
+        "@com_google_googletest//:gtest",
     ],
 )
 

mediapipe/framework/deps/re2.h

@@ -0,0 +1,59 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FRAMEWORK_DEPS_RE2_H_
+#define MEDIAPIPE_FRAMEWORK_DEPS_RE2_H_
+
+#include <regex>  // NOLINT
+
+namespace mediapipe {
+
+// Implementats a subset of RE2 using std::regex_match.
+class RE2 {
+ public:
+  RE2(const std::string& pattern) : std_regex_(pattern) {}
+  static bool FullMatch(std::string text, const RE2& re) {
+    return std::regex_match(text, re.std_regex_);
+  }
+  static bool PartialMatch(std::string text, const RE2& re) {
+    return std::regex_search(text, re.std_regex_);
+  }
+  static int GlobalReplace(std::string* text, const RE2& re,
+                           const std::string& rewrite) {
+    std::smatch sm;
+    std::regex_search(*text, sm, re.std_regex_);
+    *text = std::regex_replace(*text, re.std_regex_, rewrite);
+    return std::max(0, static_cast<int>(sm.size()) - 1);
+  }
+
+ private:
+  std::regex std_regex_;
+};
+
+// Implements LazyRE2 using std::call_once.
+class LazyRE2 {
+ public:
+  RE2& operator*() const {
+    std::call_once(once_, [&]() { ptr_ = new RE2(pattern_); });
+    return *ptr_;
+  }
+  RE2* operator->() const { return &**this; }
+  const char* pattern_;
+  mutable RE2* ptr_;
+  mutable std::once_flag once_;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_DEPS_RE2_H_

mediapipe/framework/deps/status_matchers.h

@@ -15,10 +15,44 @@
 #ifndef MEDIAPIPE_DEPS_STATUS_MATCHERS_H_
 #define MEDIAPIPE_DEPS_STATUS_MATCHERS_H_
 
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include "mediapipe/framework/deps/status.h"
 
-#define MP_EXPECT_OK(statement) EXPECT_TRUE((statement).ok())
-#define MP_ASSERT_OK(statement) ASSERT_TRUE((statement).ok())
+namespace mediapipe {
+
+// Monomorphic implementation of matcher IsOk() for a given type T.
+// T can be Status, StatusOr<>, or a reference to either of them.
+template <typename T>
+class MonoIsOkMatcherImpl : public testing::MatcherInterface<T> {
+ public:
+  void DescribeTo(std::ostream* os) const override { *os << "is OK"; }
+  void DescribeNegationTo(std::ostream* os) const override {
+    *os << "is not OK";
+  }
+  bool MatchAndExplain(T actual_value,
+                       testing::MatchResultListener*) const override {
+    return actual_value.ok();
+  }
+};
+
+// Implements IsOk() as a polymorphic matcher.
+class IsOkMatcher {
+ public:
+  template <typename T>
+  operator testing::Matcher<T>() const {  // NOLINT
+    return testing::Matcher<T>(new MonoIsOkMatcherImpl<T>());
+  }
+};
+
+// Returns a gMock matcher that matches a Status or StatusOr<> which is OK.
+inline IsOkMatcher IsOk() { return IsOkMatcher(); }
+
+}  // namespace mediapipe
+
+// Macros for testing the results of functions that return mediapipe::Status or
+// mediapipe::StatusOr<T> (for any type T).
+#define MP_EXPECT_OK(expression) EXPECT_THAT(expression, mediapipe::IsOk())
+#define MP_ASSERT_OK(expression) ASSERT_THAT(expression, mediapipe::IsOk())
 
 #endif  // MEDIAPIPE_DEPS_STATUS_MATCHERS_H_

mediapipe/framework/encode_binary_proto.bzl

@@ -141,7 +141,8 @@ def _generate_proto_descriptor_set_impl(ctx):
     # order of gendir before ., is needed for the proto compiler to resolve
     # import statements that reference proto files produced by a genrule.
     ctx.actions.run(
-        inputs = all_protos.to_list() + [ctx.executable._proto_compiler],
+        inputs = all_protos,
+        tools = [ctx.executable._proto_compiler],
         outputs = [descriptor],
         executable = ctx.executable._proto_compiler,
         arguments = [

mediapipe/framework/formats/landmark.proto

@@ -26,6 +26,11 @@ message Landmark {
   optional float x = 1;
   optional float y = 2;
   optional float z = 3;
+
+  // Landmark visibility. Float score of whether landmark is visible or occluded
+  // by other obects. Depending on the model, visibility value is either a
+  // sigmoid or an argument of sigmoid.
+  optional float visibility = 4;
 }
 
 // Group of Landmark protos.
@@ -39,6 +44,7 @@ message NormalizedLandmark {
   optional float x = 1;
   optional float y = 2;
   optional float z = 3;
+  optional float visibility = 4;
 }
 
 // Group of NormalizedLandmark protos.

mediapipe/framework/formats/object_detection/BUILD

@@ -15,10 +15,10 @@
 # Description:
 #   Working with dense optical flow in mediapipe.
 
-licenses(["notice"])  # Apache 2.0
-
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
 
+licenses(["notice"])  # Apache 2.0
+
 package(default_visibility = ["//visibility:private"])
 
 proto_library(

mediapipe/framework/legacy_calculator_support.cc

@@ -16,20 +16,11 @@
 
 namespace mediapipe {
 
-#if EMSCRIPTEN_WORKAROUND_FOR_B121216479
-template <>
-CalculatorContext*
-    LegacyCalculatorSupport::Scoped<CalculatorContext>::current_ = nullptr;
-template <>
-CalculatorContract*
-    LegacyCalculatorSupport::Scoped<CalculatorContract>::current_ = nullptr;
-#else
 template <>
 thread_local CalculatorContext*
     LegacyCalculatorSupport::Scoped<CalculatorContext>::current_ = nullptr;
 template <>
 thread_local CalculatorContract*
     LegacyCalculatorSupport::Scoped<CalculatorContract>::current_ = nullptr;
-#endif  // EMSCRIPTEN_WORKAROUND_FOR_B121216479
 
 }  // namespace mediapipe

mediapipe/framework/legacy_calculator_support.h

@@ -18,29 +18,6 @@
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/calculator_contract.h"
 
-// In Emscripten builds without threading support, some member variables
-// declared "static thread_local" will not be linked correctly. To workaround
-// this we declare them only "static".
-// TODO: remove this macro and use thread_local unconditionally
-#if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__)
-#define EMSCRIPTEN_WORKAROUND_FOR_B121216479 1
-#endif
-
-namespace mediapipe {
-class GlCalculatorHelper;
-class MetalHelperLegacySupport;
-namespace aimatter {
-template <class T>
-class CachableAsyncLoadableObject;
-}
-}  // namespace mediapipe
-
-namespace xeno {
-namespace effect {
-class AssetRegistryServiceHelper;
-}  // namespace effect
-}  // namespace xeno
-
 namespace mediapipe {
 
 class LegacyCalculatorSupport {
@@ -55,12 +32,9 @@ class LegacyCalculatorSupport {
   // from that point, and the previous value is restored when execution leaves
   // that scope, as one would expect.
   //
-  // The current value is only accessible via this mechanism by a limited set
-  // of classes (listed as friends below). This is only meant to be used where
-  // backwards compatibility reasons prevent passing the CC directly.
-  //
-  // Only two specializations are allowed: Scoped<CalculatorContext> and
-  // Scoped<CalculatorContract>.
+  // This is only meant to be used where backwards compatibility reasons prevent
+  // passing the CC directly. Specifically, it can be used to access
+  // CalculatorContext and CalculatorContract from legacy calculator code.
   template <class C>
   class Scoped {
    public:
@@ -72,53 +46,26 @@ class LegacyCalculatorSupport {
     }
     ~Scoped() { current_ = saved_; }
 
+    // The current C* for this thread.
+    static C* current() { return current_; }
+
    private:
     // The value to restore after exiting this scope.
     C* saved_;
 
-    // The current C* for this thread.
-    //
     // This needs NOLINT because, when included in Objective-C++ files,
     // clang-tidy suggests using an Objective-C naming convention, which is
     // inappropriate. (b/116015736) No category specifier because of b/71698089.
-#if EMSCRIPTEN_WORKAROUND_FOR_B121216479
-    ABSL_CONST_INIT static C* current_;  // NOLINT
-#else
-    ABSL_CONST_INIT static thread_local C* current_;  // NOLINT
-#endif
-
-    static C* current() { return current_; }
-
-    // Only these classes are allowed to access the current CC via this
-    // mechanism.
-    friend class ::mediapipe::GlCalculatorHelper;
-    friend class ::mediapipe::MetalHelperLegacySupport;
-    template <class T>
-    friend class ::mediapipe::aimatter::CachableAsyncLoadableObject;
-    friend class ::xeno::effect::AssetRegistryServiceHelper;
+    //
+    // ABSL_CONST_INIT triggers b/155992786 with some versions of Clang on Apple
+    // platforms.
+#ifndef __APPLE__
+    ABSL_CONST_INIT
+#endif                                // !__APPLE__
+    static thread_local C* current_;  // NOLINT
   };
 };
 
-// We only declare this variable for two specializations of the template because
-// it is only meant to be used for these two types.
-#if EMSCRIPTEN_WORKAROUND_FOR_B121216479
-template <>
-CalculatorContext* LegacyCalculatorSupport::Scoped<CalculatorContext>::current_;
-template <>
-CalculatorContract*
-    LegacyCalculatorSupport::Scoped<CalculatorContract>::current_;
-#elif _MSC_VER
-// MSVC interprets these declarations as definitions and during linking it
-// generates an error about multiple definitions of current_.
-#else
-template <>
-thread_local CalculatorContext*
-    LegacyCalculatorSupport::Scoped<CalculatorContext>::current_;
-template <>
-thread_local CalculatorContract*
-    LegacyCalculatorSupport::Scoped<CalculatorContract>::current_;
-#endif  // EMSCRIPTEN_WORKAROUND_FOR_B121216479
-
 }  // namespace mediapipe
 
 #endif  // MEDIAPIPE_FRAMEWORK_LEGACY_CALCULATOR_SUPPORT_H_

mediapipe/framework/port/BUILD

@@ -153,6 +153,33 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "re2",
+    hdrs = [
+        "re2.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework/deps:re2",
+    ],
+)
+
+cc_library(
+    name = "gtest",
+    testonly = 1,
+    hdrs = [
+        "gmock.h",
+        "gtest.h",
+        "status_matchers.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":status_matchers",
+        "//mediapipe/framework:port",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "gtest_main",
     testonly = 1,
@@ -163,6 +190,7 @@ cc_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
+        ":status_matchers",
         "//mediapipe/framework:port",
         "//mediapipe/framework/deps:status_matchers",
         "@com_google_googletest//:gtest_main",
@@ -369,6 +397,18 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "status_matchers",
+    testonly = 1,
+    hdrs = ["status_matchers.h"],
+    # Use this library through "mediapipe/framework/port:gtest_main".
+    visibility = ["//mediapipe/framework/port:__pkg__"],
+    deps = [
+        ":status",
+        "//mediapipe/framework/deps:status_matchers",
+    ],
+)
+
 cc_library(
     name = "threadpool",
     hdrs = ["threadpool.h"],

mediapipe/framework/port/build_config.bzl

@@ -120,7 +120,7 @@ def mediapipe_proto_library(
 def mediapipe_py_proto_library(
         name,
         srcs,
-        visibility,
+        visibility = None,
         py_proto_deps = [],
         proto_deps = None,
         api_version = None,
@@ -147,7 +147,7 @@ def mediapipe_py_proto_library(
         testonly = testonly,
     ))
 
-def mediapipe_cc_proto_library(name, srcs, visibility, deps = [], cc_deps = [], testonly = 0):
+def mediapipe_cc_proto_library(name, srcs, visibility = None, deps = [], cc_deps = [], testonly = 0):
     """Generate cc_proto_library for mediapipe open source version.
 
       Args:

mediapipe/framework/port/re2.h

@@ -0,0 +1,6 @@
+#ifndef MEDIAPIPE_FRAMEWORK_PORT_RE2_H_
+#define MEDIAPIPE_FRAMEWORK_PORT_RE2_H_
+
+#include "mediapipe/framework/deps/re2.h"
+
+#endif  // MEDIAPIPE_FRAMEWORK_PORT_RE2_H_

mediapipe/framework/profiler/BUILD

@@ -298,3 +298,24 @@ cc_library(
         "//mediapipe:apple": [],
     }),
 )
+
+cc_test(
+    name = "reporter_test",
+    srcs = ["reporter_test.cc"],
+    data = [
+        "//mediapipe/framework/profiler/testdata:mediapipe_profile_graphs",
+    ],
+    visibility = ["//visibility:private"],
+    deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_profile_cc_proto",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/port:advanced_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/profiler/reporter:reporter_lib",
+        "//mediapipe/framework/tool:test_util",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+    ],
+)

mediapipe/framework/profiler/reporter/BUILD

@@ -0,0 +1,62 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+licenses(["notice"])  # Apache 2.0
+
+cc_library(
+    name = "reporter_lib",
+    srcs = [
+        "reporter.cc",
+        "statistic.cc",
+    ],
+    hdrs = [
+        "reporter.h",
+        "statistic.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_profile_cc_proto",
+        "//mediapipe/framework/port:advanced_proto",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:re2",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "@com_google_absl//absl/container:btree",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
+cc_binary(
+    name = "print_profile",
+    srcs = ["print_profile.cc"],
+    deps = [
+        ":reporter_lib",
+        "//mediapipe/framework/port:advanced_proto",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/container:btree",
+        "@com_google_absl//absl/flags:flag",
+        "@com_google_absl//absl/flags:parse",
+        "@com_google_absl//absl/flags:usage",
+    ],
+)

mediapipe/framework/profiler/reporter/README.md

@@ -0,0 +1,102 @@
+# Command Line Profiler
+
+Allows a user to analyze trace files generated by MediaPipe from the command
+line. If you would prefer to see this information visually (or if you can't
+build the tool), you can see the same information within viz.mediapipe.dev.
+For more information on this, see [Profile Visualization](https://docs.google.com/document/d/1inBoRzKDyKEjtVws8Seceoa0xRU3mqpPgM_myDBilS4/edit#heading=h.bnft45odm046) at go/mediapipe-profiler-guide
+
+See go/mediapipe-profiler-guide for a detailed user's guide.
+
+---
+
+## Building and Executing
+
+Navigate to:
+
+    //mediapipe/framework/profiler/reporter
+
+To run:
+
+    bazel run :print_profile
+
+To test:
+
+    bazel test :reporter_test
+
+To print out the time columns and the total columns (as an example):
+
+    bazel run :print_profile -- --cols "*time*,*total*" --logfiles "<path-to-log>,<path-to-another-log>"
+
+
+---
+
+### print_profile [OPTION]... [FILE]...
+> Extract information from a set of MediaPipe trace files.
+
+**--compact**
+> print_profile will create lanes for each column, adding white space so that
+everything is easily readable. This option trims out any extra whitespace.
+
+**--cols**
+> Column separated set of columns to be shown. Omit to show everything. The user
+can use asterisks to match zero or more characters, or question marks to match a
+single character. (e.g., "time_*" will match "time_mean", "time_stddev", and so
+on).  "calculator" is always shown first.
+
+> Columns are listed below.
+
+#### Calculator Columns:
+
+**calculator**
+> The name of the calculator
+
+**fps**
+> The number of frames that this calculator can generate each second, on
+average. 1 / (latency_mean + time_mean) (Units are 1 / second).
+
+**frequency**
+
+> The rate that this calculator was asked to process packets per second.
+(Computed by # of calls total / (last_call_time - first_call_time)).  (Units are
+1 / second)
+
+**counter**
+> Number of times process() was called on the calculator. If the calculator
+generated outputs, then it will be reflected in "completed." If it did not, then
+this will be reflected in "dropped".
+
+**dropped**
+> Number of times the calculator was called but did not produce an output.
+
+**completed**
+> Number of times that this calculator was asked to process inputs after which
+it generated outputs.
+
+**processing_rate**
+> 1E+6 / time_mean.  The number of times per second this calculator could run
+process, on average.  (Units are 1 / second).
+
+**thread_count**
+> The number of threads that made use of each calculator.
+
+**time_mean**
+> Average time spent within a calculator (in microseconds).
+
+**time_stddev**
+> Standard deviation of time_mean (in microseconds).
+
+**time_total**
+> Total time spent within a calculator (in microseconds).
+
+**time_percent**
+> Percent of total time spent within a calculator.
+
+**input_latency_mean**
+> Average latency between earliest input packet and when calculator actually
+starts processing (in microseconds).
+
+**input_latency_stddev**
+> Standard deviation of input_latency_mean (in microseconds).
+
+**input_latency_total**
+> Total accumulated input_latency (in microseconds).

mediapipe/framework/profiler/reporter/print_profile.cc

@@ -0,0 +1,69 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// This program takes one input file and encodes its contents as a C++
+// std::string, which can be included in a C++ source file. It is similar to
+// filewrapper (and borrows some of its code), but simpler.
+
+#include <algorithm>
+#include <fstream>
+
+#include "absl/container/btree_map.h"
+#include "absl/flags/flag.h"
+#include "absl/flags/parse.h"
+#include "absl/flags/usage.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/commandlineflags.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/profiler/reporter/reporter.h"
+
+ABSL_FLAG(std::vector<std::string>, logfiles, {},
+          "comma-separated list of .binarypb files to process.");
+ABSL_FLAG(std::vector<std::string>, cols, {"*"},
+          "comma-separated list of columns to show. Suffix wildcards, '*', '?' "
+          "allowed.");
+ABSL_FLAG(bool, compact, false,
+          "if true, then don't print unnecessary whitespace.");
+
+using ::mediapipe::reporter::Reporter;
+
+// The command line utility to mine trace files of useful statistics to
+// determine bottlenecks and performance of a graph.
+int main(int argc, char** argv) {
+  absl::SetProgramUsageMessage("Display statistics from MediaPipe log files.");
+  absl::ParseCommandLine(argc, argv);
+
+  Reporter reporter;
+  reporter.set_compact(absl::GetFlag(FLAGS_compact));
+  const auto result = reporter.set_columns(absl::GetFlag(FLAGS_cols));
+  if (result.message().length()) {
+    std::cout << "WARNING" << std::endl << result.message();
+  }
+
+  const auto& flags_logfiles = absl::GetFlag(FLAGS_logfiles);
+  for (const auto& file_name : flags_logfiles) {
+    std::ifstream ifs(file_name.c_str(), std::ifstream::in);
+    mediapipe::proto_ns::io::IstreamInputStream isis(&ifs);
+    mediapipe::proto_ns::io::CodedInputStream coded_input_stream(&isis);
+    mediapipe::GraphProfile proto;
+    if (!proto.ParseFromCodedStream(&coded_input_stream)) {
+      std::cerr << "Failed to parse proto.\n";
+    } else {
+      reporter.Accumulate(proto);
+    }
+  }
+  reporter.Report()->Print(std::cout);
+  return 1;
+}

mediapipe/framework/profiler/reporter/reporter.cc

@@ -0,0 +1,448 @@
+#include "mediapipe/framework/profiler/reporter/reporter.h"
+
+#include <algorithm>
+#include <iterator>
+#include <memory>
+#include <ostream>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "absl/container/btree_map.h"
+#include "absl/status/status.h"
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
+#include "fstream"
+#include "map"
+#include "mediapipe/framework/calculator_profile.pb.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/re2.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+
+namespace mediapipe {
+namespace reporter {
+
+const LazyRE2 kValidColumnRegex = {"^[a-zA-Z0-9_?*]+$"};
+const LazyRE2 kReplace1WildcharRegex = {"\\?"};
+const LazyRE2 kReplace0toNWildcharRegex = {"\\*"};
+
+std::string ToStringF(double d) { return absl::StrFormat("%1.2f", d); }
+std::string ToString(double d) { return absl::StrFormat("%1.0f", d); }
+
+absl::btree_map<std::string,
+                std::function<const std::string(const CalculatorData&)>>
+    kColumns = {
+        {"calculator",
+         [](const CalculatorData& d) -> const std::string { return d.name; }},
+        {"counter",
+         [](const CalculatorData& d) -> const std::string {
+           return ToString(d.counter);
+         }},
+        {"completed",
+         [](const CalculatorData& d) -> const std::string {
+           return ToString(d.completed);
+         }},
+        {"dropped",
+         [](const CalculatorData& d) -> const std::string {
+           return ToString(d.dropped);
+         }},
+        {"fps",
+         [](const CalculatorData& d) -> const std::string {
+           return ToStringF(d.fps);
+         }},
+        {"frequency",
+         [](const CalculatorData& d) -> const std::string {
+           return ToStringF(d.frequency);
+         }},
+        {"processing_rate",
+         [](const CalculatorData& d) -> const std::string {
+           return ToStringF(d.processing_rate);
+         }},
+        {"thread_count",
+         [](const CalculatorData& d) -> const std::string {
+           return ToStringF(d.thread_count);
+         }},
+        {"time_mean",
+         [](const CalculatorData& d) -> const std::string {
+           return ToStringF(d.time_stat.mean());
+         }},
+        {"time_stddev",
+         [](const CalculatorData& d) -> const std::string {
+           return ToStringF(d.time_stat.stddev());
+         }},
+        {"time_total",
+         [](const CalculatorData& d) -> const std::string {
+           return ToString(d.time_stat.total());
+         }},
+        {"time_percent",
+         [](const CalculatorData& d) -> const std::string {
+           return ToStringF(d.time_percent);
+         }},
+        {"input_latency_mean",
+         [](const CalculatorData& d) -> const std::string {
+           return ToStringF(d.input_latency_stat.mean());
+         }},
+        {"input_latency_stddev",
+         [](const CalculatorData& d) -> const std::string {
+           return ToStringF(d.input_latency_stat.stddev());
+         }},
+        {"input_latency_total",
+         [](const CalculatorData& d) -> const std::string {
+           return ToString(d.input_latency_stat.total());
+         }}};
+
+// Holds calculator traces that have an output trace with a provided stream ID
+// and Packet timestamp.
+typedef std::map<std::pair<int64_t, int32_t>,
+                 const mediapipe::GraphTrace::CalculatorTrace*>
+    PacketKeyToCalcTrace;
+
+// Use this to locate the calculator trace with the start time for a given
+// node its thread_id and its packet timestamp.
+typedef std::map<std::pair<int64_t, std::pair<int32_t, int32_t>>,
+                 const mediapipe::GraphTrace::CalculatorTrace*>
+    TimestampNodeIdToCalcTrace;
+
+// Maps node IDs to names.
+typedef std::map<int32_t, std::string> NameLookup;
+
+Reporter::Reporter() { MEDIAPIPE_CHECK_OK(set_columns({"*"})); }
+
+int64_t RecursePacketStartTime(
+    const PacketKeyToCalcTrace& output_trace_lookup,
+    const mediapipe::GraphProfile& profile,
+    const mediapipe::GraphTrace::CalculatorTrace& trace,
+    std::vector<int32_t>* visited_calculators) {
+  int64_t child_start_time =
+      trace.has_start_time() ? trace.start_time() : trace.finish_time();
+  const int32_t node_id = trace.node_id();
+  if (std::find(visited_calculators->begin(), visited_calculators->end(),
+                node_id) != visited_calculators->end()) {
+    return child_start_time;
+  }
+  visited_calculators->push_back(node_id);
+
+  for (size_t index = 0; index < trace.input_trace_size(); ++index) {
+    const auto& stream_trace = trace.input_trace(index);
+
+    // Find the output corresponding to this input.
+    const auto it = output_trace_lookup.find(std::make_pair(
+        stream_trace.packet_timestamp(), stream_trace.stream_id()));
+    if (it != output_trace_lookup.end()) {
+      child_start_time =
+          std::min(child_start_time,
+                   RecursePacketStartTime(output_trace_lookup, profile,
+                                          *it->second, visited_calculators));
+    }
+  }
+  return child_start_time;
+}
+
+int64_t CalculateInputLatency(
+    const PacketKeyToCalcTrace& output_trace_lookup,
+    const mediapipe::GraphProfile& profile,
+    const mediapipe::GraphTrace::CalculatorTrace& trace) {
+  // Track visited calculators to detect loops.
+  std::vector<int> visited_calculators;
+
+  // If a calculator has no start time, then there is no latency to measure.
+  const auto result =
+      !trace.has_start_time()
+          ? 0L
+          : trace.start_time() - RecursePacketStartTime(output_trace_lookup,
+                                                        profile, trace,
+                                                        &visited_calculators);
+
+  return result;
+}
+
+void CacheNodeNameLookup(const mediapipe::GraphProfile& profile,
+                         NameLookup* result) {
+  for (const auto& graph_trace : profile.graph_trace()) {
+    int key = 0;
+    for (const auto& calc_name : graph_trace.calculator_name()) {
+      (*result)[key] = calc_name;
+      ++key;
+    }
+  }
+}
+
+void CacheOutputTraceLookup(const mediapipe::GraphProfile& profile,
+                            PacketKeyToCalcTrace* output_trace_lookup,
+                            TimestampNodeIdToCalcTrace* start_time_lookup) {
+  for (const auto& graph_trace : profile.graph_trace()) {
+    for (const auto& calc_trace : graph_trace.calculator_trace()) {
+      if (calc_trace.event_type() != mediapipe::GraphTrace_EventType_PROCESS) {
+        continue;
+      }
+      if (calc_trace.has_start_time() && !calc_trace.has_finish_time()) {
+        (*start_time_lookup)[std::make_pair(
+            calc_trace.input_timestamp(),
+            std::make_pair(calc_trace.node_id(), calc_trace.thread_id()))] =
+            &calc_trace;
+      }
+      for (const auto& stream_trace : calc_trace.output_trace()) {
+        (*output_trace_lookup)[std::make_pair(stream_trace.packet_timestamp(),
+                                              stream_trace.stream_id())] =
+            &calc_trace;
+      }
+    }
+  }
+}
+
+void CompleteCalculatorData(
+    const GraphData& graph_data,
+    std::map<std::string, CalculatorData>* calculator_data) {
+  for (auto& calc_entry : *calculator_data) {
+    auto& calc_data = calc_entry.second;
+    const auto& time_stat = calc_data.time_stat;
+    const auto& latency_stat = calc_data.input_latency_stat;
+    const auto time_to_process = time_stat.mean() + latency_stat.mean();
+    calc_data.fps = time_to_process == 0 ? 0 : 1.0E+6 / time_to_process;
+
+    const auto duration = graph_data.max_time - graph_data.min_time;
+    calc_data.frequency = calc_data.completed / (duration / 1.0E+6);
+    calc_data.time_percent = 100 * time_stat.total() / graph_data.total_time;
+    calc_data.dropped = calc_data.counter - calc_data.completed;
+
+    calc_data.processing_rate = calc_data.time_stat.mean() == 0
+                                    ? 0
+                                    : 1.0 / calc_data.time_stat.mean() * 1.0E+6;
+    calc_data.thread_count = calc_data.threads.size();
+  }
+}
+
+void Reporter::Accumulate(const mediapipe::GraphProfile& profile) {
+  // Cache nodeID to its std::string name.
+  NameLookup name_lookup;
+  CacheNodeNameLookup(profile, &name_lookup);
+
+  // Cache some lookups so that we can quickly find the matching output stream
+  // for a given input stream, and so that we can find the start time of a
+  // given timestamp of a node.
+  PacketKeyToCalcTrace output_trace_lookup;
+  TimestampNodeIdToCalcTrace start_event_lookup;
+  CacheOutputTraceLookup(profile, &output_trace_lookup, &start_event_lookup);
+
+  // Hold the domain of all times found in the trace file.
+  auto& min_time = graph_data_.min_time;
+  auto& max_time = graph_data_.max_time;
+  auto& total_time = graph_data_.total_time;
+
+  // The start and finish time of PROCESS events might be split between
+  // events. If a start event has been seen, we'll record it so that we can
+  // match it up later.
+  std::map<int32_t, absl::optional<int64_t>> start_times;
+
+  for (const auto& graph_trace : profile.graph_trace()) {
+    for (const auto& calc_trace : graph_trace.calculator_trace()) {
+      if (calc_trace.event_type() != mediapipe::GraphTrace_EventType_PROCESS) {
+        continue;
+      }
+
+      const auto& node_name = name_lookup[calc_trace.node_id()];
+      auto& calc_data = calculator_data_[node_name];
+
+      calc_data.name = node_name;
+      calc_data.threads.insert(calc_trace.thread_id());
+
+      // If there is a start time, update the domain of the trace time, and
+      // mark that we've seen a start time for this calculator.
+      if (calc_trace.has_start_time()) {
+        min_time =
+            std::min(min_time, static_cast<int64_t>(calc_trace.start_time() +
+                                                    graph_trace.base_time()));
+        ++calc_data.counter;
+      }
+
+      // If there is a finish time, update the domain and mark that an event
+      // has been completed.
+      if (calc_trace.has_finish_time()) {
+        const auto finish_time =
+            calc_trace.finish_time() + graph_trace.base_time();
+        max_time = std::max(max_time, static_cast<int64_t>(finish_time));
+
+        absl::optional<int64_t> start_time;
+        if (calc_trace.has_start_time()) {
+          start_time.emplace(calc_trace.start_time());
+        } else {
+          const auto start_event_it = start_event_lookup.find(std::make_pair(
+              calc_trace.input_timestamp(),
+              std::make_pair(calc_trace.node_id(), calc_trace.thread_id())));
+          if (start_event_it != start_event_lookup.end()) {
+            start_time.emplace(start_event_it->second->start_time());
+          }
+        }
+
+        // Edge case -- If a finish time came in without a start time, then
+        // we know that an event started before the trace became available.
+        // But since we don't know when that is, we can't record its duration
+        // and won't count it.
+        if (start_time) {
+          ++calc_data.completed;
+          // Add up the duration of the events that led up to this start
+          // event.
+          const auto input_latency =
+              CalculateInputLatency(output_trace_lookup, profile, calc_trace);
+          calc_data.input_latency_stat.Push(input_latency);
+          const auto duration =
+              finish_time - (start_time.value() + graph_trace.base_time());
+          calc_data.time_stat.Push(duration);
+          total_time += duration;
+        }
+      }
+    }
+  }
+}
+
+::mediapipe::Status Reporter::set_columns(
+    const std::vector<std::string>& columns) {
+  bool error = false;
+  std::stringstream warnings;
+  std::vector<std::string> new_columns({"calculator"});
+
+  // Iterate through the desired columns and build a regex.
+  for (const auto& column_matcher : columns) {
+    if (!RE2::PartialMatch(column_matcher, *kValidColumnRegex)) {
+      warnings << "Column '" << column_matcher << "' is invalid." << std::endl;
+      error = true;
+      continue;
+    }
+    std::string colString = column_matcher;
+    RE2::GlobalReplace(&colString, *kReplace0toNWildcharRegex, ".*");
+    RE2::GlobalReplace(&colString, *kReplace1WildcharRegex, ".");
+
+    // Iterator through our available columns and add them to our collection
+    // of new columns if they do not already exist.
+    bool matched = false;
+    for (const auto& column : kColumns) {
+      if (RE2::FullMatch(column.first, colString)) {
+        matched = true;
+        if (std::find(new_columns.begin(), new_columns.end(), column.first) ==
+            new_columns.end()) {
+          new_columns.push_back(column.first);
+        }
+      }
+    }
+    if (!matched) {
+      warnings << "Column '" << column_matcher << "' did not match any columns."
+               << std::endl;
+      error = true;
+    }
+  }
+  // As long as there is still one column, honor the request, even if error.
+  if (!new_columns.empty()) {
+    columns_.swap(new_columns);
+  }
+  if (!error) {
+    return ::mediapipe::OkStatus();
+  }
+  return ::mediapipe::InvalidArgumentError(warnings.str());
+}
+
+class ReportImpl : public Report {
+ public:
+  ReportImpl(const std::map<std::string, CalculatorData>& calculator_data,
+             const GraphData& graph_data)
+      : calculator_data_(calculator_data), graph_data_(graph_data) {}
+  void Print(std::ostream& output) override;
+  const std::vector<std::string>& headers() override { return headers_impl; }
+  const std::vector<std::vector<std::string>>& lines() override {
+    return lines_impl;
+  }
+  const GraphData& graph_data() override { return graph_data_; }
+  const std::map<std::string, CalculatorData>& calculator_data() override {
+    return calculator_data_;
+  }
+
+  // Each header name in alphabetical order, except the first column, which is
+  // always "calculator".
+  std::vector<std::string> headers_impl;
+
+  // Values for each calculator, corresponding to the label in headers().
+  std::vector<std::vector<std::string>> lines_impl;
+
+  // The longest std::string of any value in a given column (including the
+  // header for that column). Used for formatting the output.
+  std::vector<size_t> char_counts_impl;
+  bool compact_flag = false;
+
+  const std::map<std::string, CalculatorData>& calculator_data_;
+  const GraphData& graph_data_;
+};
+
+void ReportImpl::Print(std::ostream& output) {
+  // Print the results, but aside from the last column, add whitespace to
+  // fill space up to char_counts[column] + 1. The strings in the output
+  // are mutable to support padding, hence no const in the for loops.
+  int column_number = 0;
+  // Make a copy of the column std::string because we might be adding spaces.
+  for (auto column : headers_impl) {
+    int padding_needed = char_counts_impl[column_number] + 1 - column.length();
+    if (compact_flag) {
+      padding_needed = 1;
+    }
+    column.append(padding_needed, ' ');
+    output << column;
+    column_number++;
+  }
+  output << std::endl;
+
+  for (auto& row : lines_impl) {
+    int column_number = 0;
+    for (auto column : row) {
+      int padding_needed =
+          char_counts_impl[column_number] + 1 - column.length();
+      if (compact_flag) {
+        padding_needed = 1;
+      }
+      column.append(padding_needed, ' ');
+      output << column;
+      column_number++;
+    }
+    output << std::endl;
+  }
+}
+
+std::unique_ptr<Report> Reporter::Report() {
+  CompleteCalculatorData(graph_data_, &calculator_data_);
+
+  auto report = std::make_unique<ReportImpl>(calculator_data_, graph_data_);
+  report->compact_flag = compact_flag_;
+
+  // First row contains the column headers.
+  auto& headers = report->headers_impl;
+  auto& lines = report->lines_impl;
+  auto& char_counts = report->char_counts_impl;
+
+  headers = columns_;
+  char_counts.resize(headers.size());
+  std::transform(headers.begin(), headers.end(), char_counts.begin(),
+                 [](const auto& header) { return header.size(); });
+
+  for (const auto& header : headers) {
+    size_t line_num = 0;
+    for (auto calc_it = calculator_data_.begin();
+         calc_it != calculator_data_.end(); ++calc_it) {
+      const std::string value = kColumns[header](calc_it->second);
+      if (calc_it->second.name.empty()) {
+        continue;
+      }
+      while (line_num >= lines.size()) {
+        lines.push_back({});
+      }
+      auto& line = lines[line_num];
+      line.push_back(value);
+      const size_t char_count_index = line.size() - 1;
+      char_counts[char_count_index] =
+          std::max(char_counts[char_count_index], value.length());
+      ++line_num;
+    }
+  }
+  return report;
+}
+
+}  // namespace reporter
+}  // namespace mediapipe

mediapipe/framework/profiler/reporter/reporter.h

@@ -0,0 +1,130 @@
+#ifndef MEDIAPIPE_FRAMEWORK_PROFILER_REPORTER_REPORTER_H_
+#define MEDIAPIPE_FRAMEWORK_PROFILER_REPORTER_REPORTER_H_
+
+#include <memory>
+#include <ostream>
+#include <set>
+
+#include "map"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_profile.pb.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/profiler/reporter/statistic.h"
+
+namespace mediapipe {
+namespace reporter {
+
+struct GraphData {
+  int64_t min_time = std::numeric_limits<int64_t>::max();
+
+  int64_t max_time = std::numeric_limits<int64_t>::min();
+
+  int64_t total_time = 0;
+};
+
+// Holds all of the measured data for a calculator.
+struct CalculatorData {
+  // Name of the calculator.
+  std::string name;
+
+  // The number of times the calculator was detected entering a PROCESS state.
+  int counter;
+
+  // The number of times the calculator finished its process state.
+  int completed;
+
+  // The number of dropped packets.
+  double dropped;
+
+  // Specifies how quickly (1/s) this calculator can output events given its
+  // input latency and processing time.
+  double fps;
+
+  // Reports how often this calculator was called across the lifetime of the
+  // trace (1/s).
+  double frequency;
+
+  // The max number of threads used.
+  double thread_count;
+
+  // Reports how fast this calculator can run in isolation. (1/s).
+  double processing_rate;
+
+  // Percentage of the total time spent in this calculator.
+  double time_percent;
+
+  // Records the total time this calculator spent in PROCESS (microseconds).
+  Statistic time_stat;
+
+  // Records the input latency (microseconds). This is the longest time
+  // it takes for the input packets of a calculator to reach this calculator
+  // from their origin.
+  Statistic input_latency_stat;
+
+  // The threads on which this calculator ran.
+  std::set<int> threads;
+};
+
+// A snapshot of statistics generated by Reporter.
+class Report {
+ public:
+  virtual ~Report() = default;
+
+  // Prints the data contained within the report to a given
+  // stream (e.g., std::cout).
+  virtual void Print(std::ostream& output) = 0;
+
+  // Provides the list of headers included in the report. The column
+  // "calculator" will always come first, followed by the selected
+  // columns in alphabetical order.
+  virtual const std::vector<std::string>& headers() = 0;
+
+  // Provides a line for each calculator which in turn contains the name
+  // of the calculator followed by stringified values of each column's
+  // statistics.
+  virtual const std::vector<std::vector<std::string>>& lines() = 0;
+
+  // Returns summary data for the graph. Invalidated if Report() is called again
+  // on Reporter.
+  virtual const GraphData& graph_data() = 0;
+
+  // Returns summary data for each calculator in the graph. Invalidated if
+  // Report() is called again on Reporter.
+  virtual const std::map<std::string, CalculatorData>& calculator_data() = 0;
+};
+
+// Provides a way to accumulate statistics from one or more
+// GraphProfile protobufs in order to generate a report summarizing
+// the statistics within them.
+class Reporter {
+ public:
+  Reporter();
+
+  // Adds the contents of a given profile.
+  void Accumulate(const mediapipe::GraphProfile& profile);
+
+  // Accepts names of of columns or wildcard patterns (* or ?) to
+  // select which statistics columns will be included in a generated
+  // report.
+  ::mediapipe::Status set_columns(const std::vector<std::string>& columns);
+
+  // Generates a report based on the current accumulated statistics.
+  std::unique_ptr<reporter::Report> Report();
+
+  // Set to true to remove decorative whitespace from the output.
+  void set_compact(bool value) { compact_flag_ = value; }
+
+ private:
+  bool compact_flag_ = false;
+
+  std::vector<std::string> columns_;
+
+  // Maps calculator.name -> profile information for that calculator.
+  std::map<std::string, CalculatorData> calculator_data_;
+  GraphData graph_data_;
+};
+
+}  // namespace reporter
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_PROFILER_REPORTER_REPORTER_H_

mediapipe/framework/profiler/reporter/statistic.cc

@@ -0,0 +1,46 @@
+#include "mediapipe/framework/profiler/reporter/statistic.h"
+
+#include <cmath>
+
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_profile.pb.h"
+
+namespace mediapipe {
+namespace reporter {
+
+// Pushes a single value into the statistics, updating mean and stddev.
+void Statistic::Push(double x) {
+  ++counter_;
+
+  if (counter_ == 1) {
+    mean_ = x;
+    ssd_ = 0.0;
+    total_impl_ = x;
+  } else {
+    // Implementing Welford’s algorithm for computing variance.
+    auto old_mean = mean_;
+    mean_ = mean_ + (x - mean_) / counter_;
+    ssd_ = ssd_ + (x - mean_) * (x - old_mean);
+    total_impl_ += x;
+  }
+}
+
+// Returns the number of data points used to calculator the mean and
+// stddev.
+int Statistic::data_count() const { return counter_; }
+
+// Returns the mean of the data pushed into this statistic.
+double Statistic::mean() const { return (counter_ > 0) ? mean_ : 0.0; }
+
+// Returns the variance of the data pushed into this statistic.
+double Statistic::variance() const {
+  return ((counter_ > 1) ? ssd_ / (counter_ - 1) : 0.0);
+}
+
+// Returns the standard deviation of the data pushed into this statistic.
+double Statistic::stddev() const { return std::sqrt(variance()); }
+
+double Statistic::total() const { return total_impl_; }
+
+}  // namespace reporter
+}  // namespace mediapipe

mediapipe/framework/profiler/reporter/statistic.h

@@ -0,0 +1,51 @@
+#ifndef MEDIAPIPE_FRAMEWORK_PROFILER_REPORTER_STATISTIC_H_
+#define MEDIAPIPE_FRAMEWORK_PROFILER_REPORTER_STATISTIC_H_
+
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_profile.pb.h"
+
+namespace mediapipe {
+namespace reporter {
+
+// Allows the user to push data and maintains a counter, mean, and stddev of
+// that data.
+class Statistic {
+ public:
+  Statistic() : counter_(0) {}
+
+  // Clears the current statistic.
+  void Clear() { counter_ = 0; }
+
+  // Pushes a single value into the statistic, updating mean and stddev.
+  void Push(double x);
+
+  // Returns the number of data points used to calculate the mean and stddev.
+  int data_count() const;
+
+  // Returns the mean of the data pushed into this statistic.
+  double mean() const;
+
+  // Returns the variance of the data pushed into this statistic.
+  double variance() const;
+
+  // Returns the standard deviation of the data pushed into this statistic.
+  double stddev() const;
+
+  // Returns the sum of values of this statistic.
+  double total() const;
+
+ private:
+  int counter_;
+  double total_impl_;
+
+  // Welford's algorithm allows us to keep a running standard deviation. We need
+  // to hang onto the mean and sum of squared differences in between calls to
+  // push().
+  double mean_;
+  double ssd_;
+};
+
+}  // namespace reporter
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_PROFILER_REPORTER_STATISTIC_H_

mediapipe/framework/profiler/reporter_test.cc

@@ -0,0 +1,231 @@
+// Copyright 2018 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/profiler/reporter/reporter.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_profile.pb.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/profiler/reporter/statistic.h"
+#include "mediapipe/framework/tool/test_util.h"
+
+namespace mediapipe {
+
+using ::mediapipe::reporter::Reporter;
+using ::testing::ElementsAre;
+using ::testing::HasSubstr;
+using ::testing::IsSupersetOf;
+
+void LoadGraphProfile(const std::string& path, GraphProfile* proto) {
+  int fd = open(path.c_str(), O_RDONLY);
+  if (fd == -1) {
+    LOG(ERROR) << "could not open test graph: " << path
+               << ", error: " << strerror(errno);
+    return;
+  }
+  proto_ns::io::FileInputStream input(fd);
+  bool success = proto->ParseFromZeroCopyStream(&input);
+  close(fd);
+  if (!success) {
+    LOG(ERROR) << "could not parse test graph: " << path;
+  }
+}
+
+std::unique_ptr<Reporter> loadReporter(const std::vector<std::string>& paths) {
+  auto reporter = std::make_unique<Reporter>();
+  for (const auto path : paths) {
+    GraphProfile profile;
+    LoadGraphProfile(
+        absl::StrCat(GetTestDataDir("mediapipe/framework/profiler"), path),
+        &profile);
+    reporter->Accumulate(profile);
+  }
+  return reporter;
+}
+
+TEST(Reporter, Trivial) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+}
+
+TEST(Reporter, ReportAllColumns) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+  auto report = reporter->Report();
+
+  EXPECT_THAT(report->headers(),
+              IsSupersetOf({"calculator", "time_mean", "time_stddev",
+                            "time_total", "input_latency_mean",
+                            "input_latency_stddev", "input_latency_total"}));
+  EXPECT_EQ(report->headers()[0], "calculator");
+}
+
+// Reports only the "calculator" column and one additional column using no
+// wildcards.
+TEST(Reporter, ReportOneColumn) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+  MEDIAPIPE_CHECK_OK(reporter->set_columns({"time_mean"}));
+  auto report = reporter->Report();
+  EXPECT_THAT(report->headers(), ElementsAre("calculator", "time_mean"));
+}
+
+// Reports the "calculator" column and additional columns using wildcards.
+// Notice the columns are alphabetical except for the first column, 'calendar'.
+TEST(Reporter, ReportColumnsWithWildcards) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+  MEDIAPIPE_CHECK_OK(reporter->set_columns({"*_m??n", "*l?t*cy*"}));
+  EXPECT_THAT(reporter->Report()->headers(),
+              ElementsAre("calculator", "input_latency_mean", "time_mean",
+                          "input_latency_stddev", "input_latency_total"));
+}
+
+TEST(Reporter, AggregatesAreRecorded) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+  MEDIAPIPE_CHECK_OK(reporter->set_columns({"time_*", "*latency*"}));
+  const auto& report = reporter->Report();
+  const auto& lines = report->lines();
+  EXPECT_EQ(lines.size(), 3);
+  EXPECT_THAT(lines[2],
+              ElementsAre("OpenCvWriteTextCalculator", "13823.77", "100.00",
+                          "5541.47", "1976799", "245.13", "464.27", "35054"));
+}
+
+TEST(Reporter, JoinsFiles) {
+  auto reporter = loadReporter({
+      "profile_opencv_0.binarypb",
+      "profile_opencv_1.binarypb",
+  });
+  MEDIAPIPE_CHECK_OK(reporter->set_columns({"time_*", "*latency*"}));
+  const auto& report = reporter->Report();
+  const auto& lines = report->lines();
+  EXPECT_EQ(lines.size(), 3);
+  EXPECT_THAT(lines[2],
+              ElementsAre("OpenCvWriteTextCalculator", "14707.77", "100.00",
+                          "5630.52", "3000385", "237.50", "389.35", "48449"));
+}
+
+TEST(Reporter, PrintAllColumns) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+  auto report = reporter->Report();
+
+  std::stringstream output;
+  report->Print(output);
+  output.seekp(0);
+
+  std::string header;
+  std::getline(output, header);
+  EXPECT_THAT(header,
+              AllOf(HasSubstr("calculator"), HasSubstr("input_latency_mean"),
+                    HasSubstr("input_latency_stddev"),
+                    HasSubstr("input_latency_total"), HasSubstr("time_mean"),
+                    HasSubstr("time_stddev"), HasSubstr("time_total")));
+}
+
+TEST(Reporter, CanReportBadColumns) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+  auto result = reporter->set_columns({"il[leg]al"});
+  EXPECT_EQ(result.code(), StatusCode::kInvalidArgument);
+  EXPECT_EQ(result.message(), "Column 'il[leg]al' is invalid.\n");
+}
+
+TEST(Reporter, CanReportNonMatchingColumns) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+  auto result = reporter->set_columns({"time_*", "il[leg]al"});
+  EXPECT_EQ(result.code(), StatusCode::kInvalidArgument);
+  EXPECT_EQ(result.message(), "Column 'il[leg]al' is invalid.\n");
+  // Should not affect active columns, which is currently still "*".
+  auto report = reporter->Report();
+  EXPECT_THAT(report->headers(),
+              IsSupersetOf({"calculator", "time_mean", "time_stddev"}));
+}
+
+TEST(Reporter, BadPatternsIgnored) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+  auto result = reporter->set_columns({"time_mean", "il[leg]al", "^bad"});
+  EXPECT_EQ(result.code(), StatusCode::kInvalidArgument);
+  // Can report multiple errors at once, separated by newlines.
+  EXPECT_EQ(result.message(),
+            "Column 'il[leg]al' is invalid.\n"
+            "Column '^bad' is invalid.\n");
+  // Should not affect active columns, which is currently still "*".
+  auto report = reporter->Report();
+  EXPECT_THAT(report->headers(), ElementsAre("calculator", "time_mean"));
+}
+
+TEST(Reporter, NonMatchingColumnsIgnored) {
+  auto reporter = loadReporter({"profile_opencv_0.binarypb"});
+  auto result = reporter->set_columns({"koopa*"});
+  EXPECT_EQ(result.code(), StatusCode::kInvalidArgument);
+  EXPECT_EQ(result.message(), "Column 'koopa*' did not match any columns.\n");
+}
+
+// Tests a much simpler, fabricated log where results can easily be hand
+// calculated.
+TEST(Reporter, ProcessCalculatedCorrectly) {
+  auto reporter = loadReporter({"profile_process_test.binarypb"});
+  auto report = reporter->Report();
+  EXPECT_THAT(report->calculator_data().at("ACalculator").time_percent,
+              testing::DoubleEq(75));
+  EXPECT_THAT(report->calculator_data().at("ACalculator").time_stat.mean(),
+              testing::DoubleEq(450));
+  EXPECT_THAT(report->calculator_data().at("ACalculator").time_stat.stddev(),
+              testing::DoubleNear(70.71, 0.01));
+  EXPECT_THAT(report->calculator_data().at("ACalculator").time_stat.total(),
+              testing::DoubleEq(900));
+  EXPECT_THAT(report->calculator_data().at("BCalculator").time_percent,
+              testing::DoubleEq(25));
+  EXPECT_THAT(report->calculator_data().at("BCalculator").time_stat.mean(),
+              testing::DoubleEq(300));
+  // BCalculator has only one data point, so stddev is zero.
+  EXPECT_THAT(report->calculator_data().at("BCalculator").time_stat.stddev(),
+              testing::DoubleEq(0));
+  EXPECT_THAT(report->calculator_data().at("BCalculator").time_stat.total(),
+              testing::DoubleEq(300));
+}
+
+TEST(Reporter, LatencyCalculatedCorrectly) {
+  auto reporter = loadReporter({"profile_latency_test.binarypb"});
+  auto report = reporter->Report();
+  EXPECT_THAT(
+      report->calculator_data().at("ACalculator").input_latency_stat.mean(),
+      testing::DoubleEq(150));
+  EXPECT_THAT(
+      report->calculator_data().at("ACalculator").input_latency_stat.stddev(),
+      testing::DoubleNear(70.71, 0.01));
+  EXPECT_THAT(
+      report->calculator_data().at("ACalculator").input_latency_stat.total(),
+      testing::DoubleEq(300));
+  EXPECT_THAT(
+      report->calculator_data().at("BCalculator").input_latency_stat.mean(),
+      testing::DoubleEq(750));
+  EXPECT_THAT(
+      report->calculator_data().at("BCalculator").input_latency_stat.stddev(),
+      testing::DoubleNear(212.13, 0.01));
+  EXPECT_THAT(
+      report->calculator_data().at("BCalculator").input_latency_stat.total(),
+      testing::DoubleEq(1500));
+}
+
+}  // namespace mediapipe

mediapipe/framework/subgraph.h

@@ -46,7 +46,7 @@ class Subgraph {
 
   // Returns options of a specific type.
   template <typename T>
-  static T GetOptions(Subgraph::SubgraphOptions supgraph_options) {
+  static T GetOptions(const Subgraph::SubgraphOptions& supgraph_options) {
     return tool::OptionsMap().Initialize(supgraph_options).Get<T>();
   }
 };

mediapipe/framework/tool/BUILD

@@ -601,6 +601,37 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "test_util",
+    testonly = 1,
+    srcs = ["test_util.cc"],
+    hdrs = ["test_util.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:advanced_proto",
+        "//mediapipe/framework/port:gtest",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "//mediapipe/framework/deps:no_destructor",
+        "//mediapipe/framework/port:logging",
+    ] + select({
+        "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+        "//mediapipe:android": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:ios": [],
+        "//mediapipe:macos": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+    }),
+)
+
 exports_files(
     ["build_defs.bzl"],
     visibility = ["//mediapipe/framework:__subpackages__"],

mediapipe/framework/tool/status_util.cc

@@ -61,9 +61,11 @@ namespace tool {
       }
     }
   }
-  return ::mediapipe::Status(
+  if (error_code == StatusCode::kOk) return OkStatus();
+  Status combined = ::mediapipe::Status(
       error_code,
       absl::StrCat(general_comment, "\n", absl::StrJoin(errors, "\n")));
+  return combined;
 }
 
 }  // namespace tool

mediapipe/framework/tool/status_util.h

@@ -46,9 +46,9 @@ ABSL_DEPRECATED("Use ::mediapipe::UnknownError(error_message) instead.")
 ::mediapipe::Status AddStatusPrefix(const std::string& prefix,
                                     const ::mediapipe::Status& status);
 
-// Combine a vector of ::mediapipe::Status into a single status.  If statuses
-// is empty or all statuses are OK then ::mediapipe::OkStatus() will be
-// returned.
+// Combine a vector of ::mediapipe::Status into a single composite status.
+// If statuses is empty or all statuses are OK then ::mediapipe::OkStatus()
+// will be returned.
 // This function should be considered internal to the framework.
 // TODO Move this function to somewhere with less visibility.
 ::mediapipe::Status CombinedStatus(

mediapipe/framework/tool/status_util_test.cc

@@ -27,6 +27,10 @@
 namespace mediapipe {
 namespace {
 
+using testing::ContainerEq;
+using testing::HasSubstr;
+using testing::IsEmpty;
+
 TEST(StatusTest, StatusStopIsNotOk) { EXPECT_FALSE(tool::StatusStop().ok()); }
 
 TEST(StatusTest, Prefix) {
@@ -36,13 +40,13 @@ TEST(StatusTest, Prefix) {
       ::mediapipe::StatusCode::kInvalidArgument, base_error_message);
   ::mediapipe::Status status =
       tool::AddStatusPrefix(prefix_error_message, base_status);
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(base_error_message));
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
+  EXPECT_THAT(status.ToString(), HasSubstr(base_error_message));
+  EXPECT_THAT(status.ToString(), HasSubstr(prefix_error_message));
   EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code());
 }
 
 TEST(StatusTest, CombinedStatus) {
-  std::vector< ::mediapipe::Status> errors;
+  std::vector<::mediapipe::Status> errors;
   const std::string prefix_error_message("error_with_prefix: ");
   ::mediapipe::Status status;
 
@@ -51,12 +55,12 @@ TEST(StatusTest, CombinedStatus) {
                       "error_with_this_string");
   errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument,
                       "error_with_that_string");
+  errors.back().SetPayload("test payload type",
+                           absl::Cord(absl::string_view("hello")));
   status = tool::CombinedStatus(prefix_error_message, errors);
-  EXPECT_THAT(status.ToString(),
-              testing::HasSubstr(std::string(errors[0].message())));
-  EXPECT_THAT(status.ToString(),
-              testing::HasSubstr(std::string(errors[1].message())));
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
+  EXPECT_THAT(status.ToString(), HasSubstr(std::string(errors[0].message())));
+  EXPECT_THAT(status.ToString(), HasSubstr(std::string(errors[1].message())));
+  EXPECT_THAT(status.ToString(), HasSubstr(prefix_error_message));
   EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code());
 
   errors.clear();
@@ -65,20 +69,17 @@ TEST(StatusTest, CombinedStatus) {
   errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument,
                       "error_with_that_string");
   status = tool::CombinedStatus(prefix_error_message, errors);
-  EXPECT_THAT(status.ToString(),
-              testing::HasSubstr(std::string(errors[0].message())));
-  EXPECT_THAT(status.ToString(),
-              testing::HasSubstr(std::string(errors[1].message())));
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
+  EXPECT_THAT(status.ToString(), HasSubstr(std::string(errors[0].message())));
+  EXPECT_THAT(status.ToString(), HasSubstr(std::string(errors[1].message())));
+  EXPECT_THAT(status.ToString(), HasSubstr(prefix_error_message));
   EXPECT_EQ(::mediapipe::StatusCode::kUnknown, status.code());
   errors.clear();
   errors.emplace_back(::mediapipe::StatusCode::kOk, "error_with_this_string");
   errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument,
                       "error_with_that_string");
   status = tool::CombinedStatus(prefix_error_message, errors);
-  EXPECT_THAT(status.ToString(),
-              testing::HasSubstr(std::string(errors[1].message())));
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
+  EXPECT_THAT(status.ToString(), HasSubstr(std::string(errors[1].message())));
+  EXPECT_THAT(status.ToString(), HasSubstr(prefix_error_message));
   EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code());
 
   errors.clear();

mediapipe/framework/tool/test_util.cc

@@ -0,0 +1,333 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/tool/test_util.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <memory>
+
+#include "absl/container/flat_hash_set.h"
+#include "absl/memory/memory.h"
+#include "absl/strings/match.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/deps/no_destructor.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/proto_ns.h"
+
+#ifdef __APPLE__
+#include <CoreFoundation/CoreFoundation.h>
+#elif defined(__ANDROID__)
+#include "mediapipe/util/android/file/base/helpers.h"
+#else
+#include "mediapipe/framework/port/file_helpers.h"
+#endif
+
+namespace mediapipe {
+
+namespace {
+
+// Checks if two values are equal within the specified tolerance.
+template <typename T>
+bool EqualWithTolerance(const T value1, const T value2, const T max_diff) {
+  const T diff = (value1 >= value2) ? (value1 - value2) : (value2 - value1);
+  return diff <= max_diff;
+}
+
+template <typename T>
+bool CompareDiff(const ImageFrame& image1, const ImageFrame& image2,
+                 const T max_color_diff, const T max_alpha_diff,
+                 const float max_avg_diff, std::string* error_message) {
+  // Verify image byte depth matches expected byte depth.
+  CHECK_EQ(sizeof(T), image1.ByteDepth());
+  CHECK_EQ(sizeof(T), image2.ByteDepth());
+
+  const bool return_error = error_message != nullptr;
+
+  const int width = image1.Width();
+  const int height = image1.Height();
+  const int channels1 = image1.NumberOfChannels();
+  const int channels2 = image2.NumberOfChannels();
+  const T* pixel1 = reinterpret_cast<const T*>(image1.PixelData());
+  const T* pixel2 = reinterpret_cast<const T*>(image2.PixelData());
+  const int num_channels = std::min(channels1, channels2);
+
+  // Verify the width steps are multiples of byte depth.
+  CHECK_EQ(image1.WidthStep() % image1.ByteDepth(), 0);
+  CHECK_EQ(image2.WidthStep() % image2.ByteDepth(), 0);
+  const int width_padding1 =
+      image1.WidthStep() / image1.ByteDepth() - width * channels1;
+  const int width_padding2 =
+      image2.WidthStep() / image2.ByteDepth() - width * channels2;
+
+  float avg_diff = 0;
+  uint diff_count = 0;
+  for (int row = 0; row < height; ++row) {
+    for (int col = 0; col < width; ++col) {
+      for (int channel = 0; channel < num_channels; ++channel) {
+        // Check local difference.
+        const T max_diff = channel < 3 ? max_color_diff : max_alpha_diff;
+        const T value1 = pixel1[channel];
+        const T value2 = pixel2[channel];
+        if (!EqualWithTolerance<T>(value1, value2, max_diff)) {
+          // We cast uint8 to int using this type (and leave other values as-is)
+          // to avoid printing as a single char.
+          using TypeToPrint =
+              typename std::conditional<std::is_same<T, uint8>::value, int,
+                                        T>::type;
+          std::string error = absl::Substitute(
+              "images differ: row = $0 col = $1 channel = $2 : pixel1 = $3, "
+              "pixel2 = $4",
+              row, col, channel, static_cast<TypeToPrint>(value1),
+              static_cast<TypeToPrint>(value2));
+          if (return_error) {
+            *error_message = error;
+          } else {
+            LOG(ERROR) << error;
+          }
+          return false;
+        }
+        // Check global average difference.
+        const float diff =
+            std::abs(static_cast<float>(value1) - static_cast<float>(value2));
+        avg_diff += (diff - avg_diff) / ++diff_count;
+      }
+      pixel1 += channels1;
+      pixel2 += channels2;
+    }
+    pixel1 += width_padding1;
+    pixel2 += width_padding2;
+  }
+
+  if (avg_diff > max_avg_diff) {
+    std::string error =
+        absl::Substitute("images differ: avg pixel error = $0", avg_diff);
+    if (return_error) {
+      *error_message = error;
+    } else {
+      LOG(ERROR) << error;
+    }
+    return false;
+  }
+
+  return true;
+}
+
+#if defined(__linux__)
+// Returns the directory of the running test binary.
+std::string GetBinaryDirectory() {
+  char full_path[PATH_MAX + 1];
+  int length = readlink("/proc/self/exe", full_path, PATH_MAX + 1);
+  CHECK_GT(length, 0);
+  return std::string(
+      ::mediapipe::file::Dirname(absl::string_view(full_path, length)));
+}
+#endif
+
+}  // namespace
+
+bool CompareImageFrames(const ImageFrame& image1, const ImageFrame& image2,
+                        const float max_color_diff, const float max_alpha_diff,
+                        const float max_avg_diff, std::string* error_message) {
+  const bool return_error = error_message != nullptr;
+
+  auto IsSupportedImageFormatComparison = [](const ImageFrame& image1,
+                                             const ImageFrame& image2) {
+    // Pairs of non-equal image formats that can be compared against each other.
+    static const mediapipe::NoDestructor<absl::flat_hash_set<
+        std::pair<ImageFormat::Format, ImageFormat::Format>>>
+        kCompatibleImageFormats({
+            {ImageFormat::SRGB, ImageFormat::SRGBA},
+            {ImageFormat::SRGB48, ImageFormat::SRGBA64},
+        });
+
+    auto* compatible_image_formats = kCompatibleImageFormats.get();
+
+    return image1.Format() == image2.Format() ||
+           compatible_image_formats->contains(
+               {image1.Format(), image2.Format()}) ||
+           compatible_image_formats->contains(
+               {image2.Format(), image1.Format()});
+  };
+
+  if (!IsSupportedImageFormatComparison(image1, image2)) {
+    std::string error = absl::Substitute(
+        "unsupported image format comparison; image1 = $0, image2 = $1",
+        image1.Format(), image2.Format());
+    if (return_error) {
+      *error_message = error;
+    } else {
+      LOG(ERROR) << error;
+    }
+    return false;
+  }
+
+  if (image1.Width() != image2.Width()) {
+    std::string error =
+        absl::Substitute("image width mismatch: image1 = $0, image2 = $1",
+                         image1.Width(), image2.Width());
+    if (return_error) {
+      *error_message = error;
+    } else {
+      LOG(ERROR) << error;
+    }
+    return false;
+  }
+
+  if (image1.Height() != image2.Height()) {
+    std::string error =
+        absl::Substitute("image height mismatch: image1 = $0, image2 = $1",
+                         image1.Height(), image2.Height());
+    if (return_error) {
+      *error_message = error;
+    } else {
+      LOG(ERROR) << error;
+    }
+    return false;
+  }
+
+  if (image1.ByteDepth() != image2.ByteDepth()) {
+    std::string error =
+        absl::Substitute("image byte depth mismatch: image1 = $0, image2 = $1",
+                         image1.ByteDepth(), image2.ByteDepth());
+    if (return_error) {
+      *error_message = error;
+    } else {
+      LOG(ERROR) << error;
+    }
+    return false;
+  }
+
+  switch (image1.Format()) {
+    case ImageFormat::GRAY8:
+    case ImageFormat::SRGB:
+    case ImageFormat::SRGBA:
+    case ImageFormat::LAB8:
+      return CompareDiff<uint8>(image1, image2, max_color_diff, max_alpha_diff,
+                                max_avg_diff, error_message);
+    case ImageFormat::GRAY16:
+    case ImageFormat::SRGB48:
+    case ImageFormat::SRGBA64:
+      return CompareDiff<uint16>(image1, image2, max_color_diff, max_alpha_diff,
+                                 max_avg_diff, error_message);
+    case ImageFormat::VEC32F1:
+    case ImageFormat::VEC32F2:
+      return CompareDiff<float>(image1, image2, max_color_diff, max_alpha_diff,
+                                max_avg_diff, error_message);
+    default:
+      LOG(FATAL) << ImageFrame::InvalidFormatString(image1.Format());
+  }
+}
+
+std::string GetTestRootDir() {
+#ifdef __APPLE__
+  char path[1024];
+  CFURLRef bundle_url = CFBundleCopyBundleURL(CFBundleGetMainBundle());
+  Boolean success = CFURLGetFileSystemRepresentation(
+      bundle_url, true, reinterpret_cast<UInt8*>(path), sizeof(path));
+  CHECK(success);
+  CFRelease(bundle_url);
+  return path;
+#elif defined(__ANDROID__)
+  char path[1024];
+  char* ptr = getcwd(path, sizeof(path));
+  CHECK_EQ(ptr, path);
+  return path;
+#else
+  return ::mediapipe::file::JoinPath(std::getenv("TEST_SRCDIR"), "mediapipe");
+#endif  // defined(__APPLE__)
+}
+
+std::string GetTestDataDir(const std::string& package_base_path) {
+#ifdef __APPLE__
+  return ::mediapipe::file::JoinPath(GetTestRootDir(), "testdata/");
+#elif defined(__ANDROID__)
+  std::string data_dir = GetTestRootDir();
+  std::string binary_dir = GetBinaryDirectory();
+  // In Mobile Harness, the cwd is "/" and the run dir is "/data/local/tmp".
+  if (data_dir == "/" && absl::StartsWith(binary_dir, "/data")) {
+    data_dir = binary_dir;
+  }
+  return ::mediapipe::file::JoinPath(data_dir, package_base_path, "testdata/");
+#else
+  return ::mediapipe::file::JoinPath(GetTestRootDir(), package_base_path,
+                                     "testdata/");
+#endif  // defined(__APPLE__)
+}
+
+std::unique_ptr<ImageFrame> LoadTestPng(const std::string& path,
+                                        ImageFormat::Format format) {
+  return nullptr;
+}
+
+bool LoadTestGraph(CalculatorGraphConfig* proto, const std::string& path) {
+  int fd = open(path.c_str(), O_RDONLY);
+  if (fd == -1) {
+    LOG(ERROR) << "could not open test graph: " << path
+               << ", error: " << strerror(errno);
+    return false;
+  }
+  proto_ns::io::FileInputStream input(fd);
+  bool success = proto->ParseFromZeroCopyStream(&input);
+  close(fd);
+  if (!success) {
+    LOG(ERROR) << "could not parse test graph: " << path;
+  }
+  return success;
+}
+
+std::unique_ptr<ImageFrame> GenerateLuminanceImage(
+    const ImageFrame& original_image) {
+  const int width = original_image.Width();
+  const int height = original_image.Height();
+  const int channels = original_image.NumberOfChannels();
+  if (channels != 3 && channels != 4) {
+    LOG(ERROR) << "Invalid number of image channels: " << channels;
+    return nullptr;
+  }
+  auto luminance_image =
+      absl::make_unique<ImageFrame>(original_image.Format(), width, height,
+                                    ImageFrame::kGlDefaultAlignmentBoundary);
+  const uint8* pixel1 = original_image.PixelData();
+  uint8* pixel2 = luminance_image->MutablePixelData();
+  const int width_padding1 = original_image.WidthStep() - width * channels;
+  const int width_padding2 = luminance_image->WidthStep() - width * channels;
+  for (int row = 0; row < height; ++row) {
+    for (int col = 0; col < width; ++col) {
+      float luminance =
+          pixel1[0] * 0.2125f + pixel1[1] * 0.7154f + pixel1[2] * 0.0721f;
+      uint8 luminance_byte = 255;
+      if (luminance < 255.0f) {
+        luminance_byte = static_cast<uint8>(luminance);
+      }
+      pixel2[0] = luminance_byte;
+      pixel2[1] = luminance_byte;
+      pixel2[2] = luminance_byte;
+      if (channels == 4) {
+        pixel2[3] = pixel1[3];
+      }
+      pixel1 += channels;
+      pixel2 += channels;
+    }
+    pixel1 += width_padding1;
+    pixel2 += width_padding2;
+  }
+  return luminance_image;
+}
+
+}  // namespace mediapipe

mediapipe/framework/tool/test_util.h

@@ -0,0 +1,67 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FRAMEWORK_TEST_UTIL_H_
+#define MEDIAPIPE_FRAMEWORK_TEST_UTIL_H_
+
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+
+namespace mediapipe {
+using mediapipe::CalculatorGraphConfig;
+
+// Checks if two image frames are equal within the specified tolerance.
+// image1 and image2 may be of different-but-compatible image formats (e.g.,
+// SRGB and SRGBA); in that case, only the channels available in both are
+// compared.
+// max_color_diff applies to the first 3 channels; i.e., R, G, B for sRGB and
+// sRGBA, and the single gray channel for GRAY8 and GRAY16. It is the maximum
+// pixel color value difference allowed; i.e., a value from 0 to 2^8-1 for 8-bit
+// data and a value from 0 to 2^16-1 for 16-bit data.
+// max_alpha_diff applies to the 4th (alpha) channel only, if present.
+// max_avg_diff applies to all channels, normalized across all pixels.
+//
+// Note: Although max_color_diff and max_alpha_diff are floats, all uint8/uint16
+// values are exactly representable. (2^24 + 1 is the first non-representable
+// positive integral value.)
+bool CompareImageFrames(const ImageFrame& image1, const ImageFrame& image2,
+                        const float max_color_diff, const float max_alpha_diff,
+                        const float max_avg_diff = 1.0,
+                        std::string* error_message = nullptr);
+
+// Returns the absolute path to the directory that contains test source code.
+std::string GetTestRootDir();
+
+// Returns the absolute path to the contents of the package's "testdata"
+// directory.
+// This handles the different paths where test data ends up when using
+// ion_cc_test on various platforms.
+std::string GetTestDataDir(const std::string& package_base_path);
+
+// Loads a binary graph from path. Returns true iff successful.
+bool LoadTestGraph(CalculatorGraphConfig* proto, const std::string& path);
+
+// Loads a PNG image from path using the given ImageFormat. Returns nullptr in
+// case of failure.
+std::unique_ptr<ImageFrame> LoadTestPng(
+    const std::string& path, ImageFormat::Format format = ImageFormat::SRGBA);
+
+// Returns the luminance image of |original_image|.
+// The format of |original_image| must be sRGB or sRGBA.
+std::unique_ptr<ImageFrame> GenerateLuminanceImage(
+    const ImageFrame& original_image);
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_TEST_UTIL_H_

mediapipe/framework/transitive_protos.bzl

@@ -5,20 +5,21 @@ ProtoLibsInfo = provider(fields = ["targets", "out"])
 def _get_proto_rules(deps, proto_rules = None):
     useful_deps = [dep for dep in deps if hasattr(dep, "proto_rules")]
     if proto_rules == None:
-        proto_rules = []
-    for dep in useful_deps:
-        proto_rules = proto_rules + dep.proto_rules
+        proto_rules = depset()
+    proto_rules = depset(
+        transitive = [proto_rules] + [dep.proto_rules for dep in useful_deps],
+    )
     return proto_rules
 
 def _proto_rules_aspect_impl(target, ctx):
     # Make sure the rule has a srcs attribute.
-    proto_rules = []
+    proto_rules = depset()
     found_cc_proto = False
     if hasattr(ctx.rule.attr, "srcs") and len(ctx.rule.attr.srcs) == 1:
         for f in ctx.rule.attr.srcs[0].files.to_list():
             if f.basename.endswith(".pb.cc"):
-                proto_rules = [target[CcInfo]]
-                found = True
+                proto_rules = depset([target[CcInfo]])
+                found_cc_proto = True
                 break
 
     if not found_cc_proto:
@@ -43,10 +44,10 @@ def _transitive_protos_impl(ctx):
       A proto provider (with transitive_sources and transitive_descriptor_sets filled in),
       and marks all transitive sources as default output.
     """
-    cc_infos = []
+    cc_info_sets = []
     for dep in ctx.attr.deps:
-        for dep_proto_rule in dep.proto_rules:
-            cc_infos.append(dep_proto_rule)
+        cc_info_sets.append(dep.proto_rules)
+    cc_infos = depset(transitive = cc_info_sets).to_list()
     return [cc_common.merge_cc_infos(cc_infos = cc_infos)]
 
 transitive_protos = rule(