From 8a2ec518deb57699163bd5eb7a5d2f4d54f71135 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 12 Jun 2023 08:42:40 -0700 Subject: [PATCH 1/9] Use .mjs for ESM Modules and use .cjs for CommonJS PiperOrigin-RevId: 539664711 --- mediapipe/tasks/web/audio/BUILD | 28 +++++++++++++++++++++++++--- mediapipe/tasks/web/package.json | 7 ++++--- mediapipe/tasks/web/text/BUILD | 28 +++++++++++++++++++++++++--- mediapipe/tasks/web/vision/BUILD | 28 +++++++++++++++++++++++++--- 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/mediapipe/tasks/web/audio/BUILD b/mediapipe/tasks/web/audio/BUILD index 4dd5a2f6..3338d17b 100644 --- a/mediapipe/tasks/web/audio/BUILD +++ b/mediapipe/tasks/web/audio/BUILD @@ -38,7 +38,7 @@ mediapipe_files(srcs = [ ]) rollup_bundle( - name = "audio_bundle", + name = "audio_bundle_mjs", config_file = "//mediapipe/tasks/web:rollup.config.mjs", entry_point = "index.ts", format = "esm", @@ -69,6 +69,29 @@ rollup_bundle( ], ) +genrule( + name = "audio_sources", + srcs = [ + ":audio_bundle_cjs", + ":audio_bundle_mjs", + ], + outs = [ + "audio_bundle.cjs", + "audio_bundle.cjs.map", + "audio_bundle.mjs", + "audio_bundle.mjs.map", + ], + cmd = ( + "for FILE in $(SRCS); do " + + " OUT_FILE=$(GENDIR)/mediapipe/tasks/web/audio/$$(" + + " basename $$FILE | sed -E 's/_([cm])js\\.js/.\\1js/'" + + " ); " + + " echo $$FILE ; echo $$OUT_FILE ; " + + " cp $$FILE $$OUT_FILE ; " + + "done;" + ), +) + genrule( name = "package_json", srcs = ["//mediapipe/tasks/web:package.json"], @@ -91,8 +114,7 @@ pkg_npm( "wasm/audio_wasm_internal.wasm", "wasm/audio_wasm_nosimd_internal.js", "wasm/audio_wasm_nosimd_internal.wasm", - ":audio_bundle", - ":audio_bundle_cjs", + ":audio_sources", ":package_json", ], ) diff --git a/mediapipe/tasks/web/package.json b/mediapipe/tasks/web/package.json index 3f495d15..4a42018f 100644 --- a/mediapipe/tasks/web/package.json +++ b/mediapipe/tasks/web/package.json @@ -2,11 +2,12 @@ "name": "@mediapipe/tasks-__NAME__", "version": "__VERSION__", "description": "__DESCRIPTION__", - "main": "__NAME___bundle_cjs.js", - "browser": "__NAME___bundle.js", - "module": "__NAME___bundle.js", + "main": "__NAME___bundle.cjs", + "browser": "__NAME___bundle.mjs", + "module": "__NAME___bundle.mjs", "author": "mediapipe@google.com", "license": "Apache-2.0", + "type": "module", "types": "__TYPES__", "homepage": "http://mediapipe.dev", "keywords": [ "AR", "ML", "Augmented", "MediaPipe", "MediaPipe Tasks" ] diff --git a/mediapipe/tasks/web/text/BUILD b/mediapipe/tasks/web/text/BUILD index f68a8c9f..76d875ba 100644 --- a/mediapipe/tasks/web/text/BUILD +++ b/mediapipe/tasks/web/text/BUILD @@ -39,7 +39,7 @@ mediapipe_ts_library( ) rollup_bundle( - name = "text_bundle", + name = "text_bundle_mjs", config_file = "//mediapipe/tasks/web:rollup.config.mjs", entry_point = "index.ts", format = "esm", @@ -70,6 +70,29 @@ rollup_bundle( ], ) +genrule( + name = "text_sources", + srcs = [ + ":text_bundle_cjs", + ":text_bundle_mjs", + ], + outs = [ + "text_bundle.cjs", + "text_bundle.cjs.map", + "text_bundle.mjs", + "text_bundle.mjs.map", + ], + cmd = ( + "for FILE in $(SRCS); do " + + " OUT_FILE=$(GENDIR)/mediapipe/tasks/web/text/$$(" + + " basename $$FILE | sed -E 's/_([cm])js\\.js/.\\1js/'" + + " ); " + + " echo $$FILE ; echo $$OUT_FILE ; " + + " cp $$FILE $$OUT_FILE ; " + + "done;" + ), +) + genrule( name = "package_json", srcs = ["//mediapipe/tasks/web:package.json"], @@ -93,7 +116,6 @@ pkg_npm( "wasm/text_wasm_nosimd_internal.js", "wasm/text_wasm_nosimd_internal.wasm", ":package_json", - ":text_bundle", - ":text_bundle_cjs", + ":text_sources", ], ) diff --git a/mediapipe/tasks/web/vision/BUILD b/mediapipe/tasks/web/vision/BUILD index a7767fe5..58795b16 100644 --- a/mediapipe/tasks/web/vision/BUILD +++ b/mediapipe/tasks/web/vision/BUILD @@ -50,7 +50,7 @@ mediapipe_ts_library( ) rollup_bundle( - name = "vision_bundle", + name = "vision_bundle_mjs", config_file = "//mediapipe/tasks/web:rollup.config.mjs", entry_point = "index.ts", format = "esm", @@ -81,6 +81,29 @@ rollup_bundle( ], ) +genrule( + name = "vision_sources", + srcs = [ + ":vision_bundle_cjs", + ":vision_bundle_mjs", + ], + outs = [ + "vision_bundle.cjs", + "vision_bundle.cjs.map", + "vision_bundle.mjs", + "vision_bundle.mjs.map", + ], + cmd = ( + "for FILE in $(SRCS); do " + + " OUT_FILE=$(GENDIR)/mediapipe/tasks/web/vision/$$(" + + " basename $$FILE | sed -E 's/_([cm])js\\.js/.\\1js/'" + + " ); " + + " echo $$FILE ; echo $$OUT_FILE ; " + + " cp $$FILE $$OUT_FILE ; " + + "done;" + ), +) + genrule( name = "package_json", srcs = ["//mediapipe/tasks/web:package.json"], @@ -104,7 +127,6 @@ pkg_npm( "wasm/vision_wasm_nosimd_internal.js", "wasm/vision_wasm_nosimd_internal.wasm", ":package_json", - ":vision_bundle", - ":vision_bundle_cjs", + ":vision_sources", ], ) From fe0d1b1e8336195e81f61edf8b14a7e5a36fc192 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 12 Jun 2023 09:25:40 -0700 Subject: [PATCH 2/9] Internal change PiperOrigin-RevId: 539675912 --- .../python/metadata/flatbuffers_lib/flatbuffers_lib.cc | 4 ++-- third_party/flatbuffers/workspace.bzl | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/python/metadata/flatbuffers_lib/flatbuffers_lib.cc b/mediapipe/tasks/python/metadata/flatbuffers_lib/flatbuffers_lib.cc index 0c251c69..cf6ddd9b 100644 --- a/mediapipe/tasks/python/metadata/flatbuffers_lib/flatbuffers_lib.cc +++ b/mediapipe/tasks/python/metadata/flatbuffers_lib/flatbuffers_lib.cc @@ -41,12 +41,12 @@ PYBIND11_MODULE(_pywrap_flatbuffers, m) { self->PushFlatBuffer(reinterpret_cast(contents.c_str()), contents.length()); }); - m.def("generate_text_file", &flatbuffers::GenerateTextFile); + m.def("generate_text_file", &flatbuffers::GenTextFile); m.def("generate_text", [](const flatbuffers::Parser& parser, const std::string& buffer) -> std::string { std::string text; - const char* result = flatbuffers::GenerateText( + const char* result = flatbuffers::GenText( parser, reinterpret_cast(buffer.c_str()), &text); if (result) { return ""; diff --git a/third_party/flatbuffers/workspace.bzl b/third_party/flatbuffers/workspace.bzl index 0edb7a6f..d06e2cbe 100644 --- a/third_party/flatbuffers/workspace.bzl +++ b/third_party/flatbuffers/workspace.bzl @@ -5,11 +5,11 @@ load("//third_party:repo.bzl", "third_party_http_archive") def repo(): third_party_http_archive( name = "flatbuffers", - strip_prefix = "flatbuffers-23.5.8", - sha256 = "55b75dfa5b6f6173e4abf9c35284a10482ba65db886b39db511eba6c244f1e88", + strip_prefix = "flatbuffers-23.5.26", + sha256 = "1cce06b17cddd896b6d73cc047e36a254fb8df4d7ea18a46acf16c4c0cd3f3f3", urls = [ - "https://github.com/google/flatbuffers/archive/v23.5.8.tar.gz", - "https://github.com/google/flatbuffers/archive/v23.5.8.tar.gz", + "https://github.com/google/flatbuffers/archive/v23.5.26.tar.gz", + "https://github.com/google/flatbuffers/archive/v23.5.26.tar.gz", ], build_file = "//third_party/flatbuffers:BUILD.bazel", delete = ["build_defs.bzl", "BUILD.bazel"], From 96cc0fd07bb4f51eda84ce2f34cf713460a9d036 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 12 Jun 2023 11:51:34 -0700 Subject: [PATCH 3/9] Internal change PiperOrigin-RevId: 539719443 --- mediapipe/framework/BUILD | 16 + .../calculator_graph_summary_packet_test.cc | 327 ++++++++++++++++++ mediapipe/framework/timestamp.cc | 7 + mediapipe/framework/timestamp.h | 4 + mediapipe/framework/timestamp_test.cc | 16 + 5 files changed, 370 insertions(+) create mode 100644 mediapipe/framework/calculator_graph_summary_packet_test.cc diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index a7d9e0a6..86608285 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -1355,6 +1355,22 @@ cc_test( ], ) +cc_test( + name = "calculator_graph_summary_packet_test", + srcs = ["calculator_graph_summary_packet_test.cc"], + deps = [ + ":calculator_framework", + ":packet", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/api2:packet", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/stream_handler:immediate_input_stream_handler", + "//mediapipe/framework/tool:sink", + ], +) + cc_test( name = "calculator_runner_test", size = "medium", diff --git a/mediapipe/framework/calculator_graph_summary_packet_test.cc b/mediapipe/framework/calculator_graph_summary_packet_test.cc new file mode 100644 index 00000000..c8d1e7eb --- /dev/null +++ b/mediapipe/framework/calculator_graph_summary_packet_test.cc @@ -0,0 +1,327 @@ +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/api2/packet.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_matchers.h" + +namespace mediapipe { + +using ::mediapipe::api2::Input; +using ::mediapipe::api2::Node; +using ::mediapipe::api2::Output; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::Value; + +namespace { + +MATCHER_P2(IntPacket, value, timestamp, "") { + *result_listener << "where object is (value: " << arg.template Get() + << ", timestamp: " << arg.Timestamp() << ")"; + return Value(arg.template Get(), Eq(value)) && + Value(arg.Timestamp(), Eq(timestamp)); +} + +// Calculates and produces sum of all passed inputs when no more packets can be +// expected on the input stream. +class SummaryPacketCalculator : public Node { + public: + static constexpr Input kIn{"IN"}; + static constexpr Output kOut{"SUMMARY"}; + + MEDIAPIPE_NODE_CONTRACT(kIn, kOut); + + static absl::Status UpdateContract(CalculatorContract* cc) { + // Makes sure there are no automatic timestamp bound updates when Process + // is called. + cc->SetTimestampOffset(TimestampDiff::Unset()); + // Currently, only ImmediateInputStreamHandler supports "done" timestamp + // bound update. (ImmediateInputStreamhandler handles multiple input + // streams differently, so, in that case, calculator adjustments may be + // required.) + // TODO: update all input stream handlers to support "done" + // timestamp bound update. + cc->SetInputStreamHandler("ImmediateInputStreamHandler"); + // Enables processing timestamp bound updates. For this use case we are + // specifically interested in "done" timestamp bound update. (E.g. when + // all input packet sources are closed.) + cc->SetProcessTimestampBounds(true); + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext* cc) final { + if (!kIn(cc).IsEmpty()) { + value_ += kIn(cc).Get(); + } + + if (kOut(cc).IsClosed()) { + // This can happen: + // 1. If, during previous invocation, kIn(cc).IsDone() == true (e.g. + // source calculator finished generating packets sent to kIn) and + // HasNextAllowedInStream() == true (which is an often case). + // 2. For Timestamp::PreStream, ImmediateInputStreamHandler will still + // invoke Process() with Timestamp::Max to indicate "Done" timestamp + // bound update. + return absl::OkStatus(); + } + + // TODO: input stream holding a packet with timestamp that has + // no next timestamp allowed in stream should always result in + // InputStream::IsDone() == true. + if (kIn(cc).IsDone() || !cc->InputTimestamp().HasNextAllowedInStream()) { + // kOut(cc).Send(value_) can be used here as well, however in the case of + // source calculator sending inputs into kIn the resulting timestamp is + // not well defined (e.g. it can be the last packet timestamp or + // Timestamp::Max()) + // TODO: last packet from source should always result in + // InputStream::IsDone() == true. + kOut(cc).Send(value_, Timestamp::Max()); + kOut(cc).Close(); + } + return absl::OkStatus(); + } + + private: + int value_ = 0; +}; +MEDIAPIPE_REGISTER_NODE(SummaryPacketCalculator); + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnClosingAllPacketSources) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: 'input' + node { + calculator: "SummaryPacketCalculator" + input_stream: 'IN:input' + output_stream: 'SUMMARY:output' + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + auto send_packet = [&graph](int value, Timestamp timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input", MakePacket(value).At(timestamp))); + }; + + send_packet(10, Timestamp(10)); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + send_packet(20, Timestamp(11)); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max()))); +} + +TEST(SummaryPacketCalculatorUseCaseTest, ProducesSummaryPacketOnMaxTimestamp) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: 'input' + node { + calculator: "SummaryPacketCalculator" + input_stream: 'IN:input' + output_stream: 'SUMMARY:output' + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + auto send_packet = [&graph](int value, Timestamp timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input", MakePacket(value).At(timestamp))); + }; + + send_packet(10, Timestamp(10)); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + send_packet(20, Timestamp::Max()); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max()))); + + output_packets.clear(); + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, IsEmpty()); +} + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnPreStreamTimestamp) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: 'input' + node { + calculator: "SummaryPacketCalculator" + input_stream: 'IN:input' + output_stream: 'SUMMARY:output' + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + auto send_packet = [&graph](int value, Timestamp timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input", MakePacket(value).At(timestamp))); + }; + + send_packet(10, Timestamp::PreStream()); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max()))); + + output_packets.clear(); + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, IsEmpty()); +} + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnPostStreamTimestamp) { + std::vector output_packets; + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie(R"pb( + input_stream: 'input' + node { + calculator: "SummaryPacketCalculator" + input_stream: 'IN:input' + output_stream: 'SUMMARY:output' + } + )pb"); + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + auto send_packet = [&graph](int value, Timestamp timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input", MakePacket(value).At(timestamp))); + }; + + send_packet(10, Timestamp::PostStream()); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max()))); + + output_packets.clear(); + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, IsEmpty()); +} + +class IntGeneratorCalculator : public Node { + public: + static constexpr Output kOut{"INT"}; + + MEDIAPIPE_NODE_CONTRACT(kOut); + + absl::Status Process(CalculatorContext* cc) final { + kOut(cc).Send(20, Timestamp(0)); + kOut(cc).Send(10, Timestamp(1000)); + return tool::StatusStop(); + } +}; +MEDIAPIPE_REGISTER_NODE(IntGeneratorCalculator); + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnSourceCalculatorCompletion) { + std::vector output_packets; + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie(R"pb( + node { + calculator: "IntGeneratorCalculator" + output_stream: "INT:int_value" + } + node { + calculator: "SummaryPacketCalculator" + input_stream: "IN:int_value" + output_stream: "SUMMARY:output" + } + )pb"); + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_EXPECT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max()))); +} + +class EmitOnCloseCalculator : public Node { + public: + static constexpr Input kIn{"IN"}; + static constexpr Output kOut{"INT"}; + + MEDIAPIPE_NODE_CONTRACT(kIn, kOut); + + absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); } + + absl::Status Close(CalculatorContext* cc) final { + kOut(cc).Send(20, Timestamp(0)); + kOut(cc).Send(10, Timestamp(1000)); + return absl::OkStatus(); + } +}; +MEDIAPIPE_REGISTER_NODE(EmitOnCloseCalculator); + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnAnotherCalculatorClosure) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: "input" + node { + calculator: "EmitOnCloseCalculator" + input_stream: "IN:input" + output_stream: "INT:int_value" + } + node { + calculator: "SummaryPacketCalculator" + input_stream: "IN:int_value" + output_stream: "SUMMARY:output" + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + MP_ASSERT_OK(graph.CloseInputStream("input")); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max()))); + + output_packets.clear(); + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, IsEmpty()); +} + +} // namespace +} // namespace mediapipe diff --git a/mediapipe/framework/timestamp.cc b/mediapipe/framework/timestamp.cc index 05b69747..4ece74c9 100644 --- a/mediapipe/framework/timestamp.cc +++ b/mediapipe/framework/timestamp.cc @@ -131,6 +131,13 @@ Timestamp Timestamp::NextAllowedInStream() const { return *this + 1; } +bool Timestamp::HasNextAllowedInStream() const { + if (*this >= Max() || *this == PreStream()) { + return false; + } + return true; +} + Timestamp Timestamp::PreviousAllowedInStream() const { if (*this <= Min() || *this == PostStream()) { // Indicates that no previous timestamps may occur. diff --git a/mediapipe/framework/timestamp.h b/mediapipe/framework/timestamp.h index 966ec183..d125d28b 100644 --- a/mediapipe/framework/timestamp.h +++ b/mediapipe/framework/timestamp.h @@ -186,6 +186,10 @@ class Timestamp { // CHECKs that this->IsAllowedInStream(). Timestamp NextAllowedInStream() const; + // Returns true if there's a next timestamp in the range [Min .. Max] after + // this one. + bool HasNextAllowedInStream() const; + // Returns the previous timestamp in the range [Min .. Max], or // Unstarted() if no Packets may preceed one with this timestamp. Timestamp PreviousAllowedInStream() const; diff --git a/mediapipe/framework/timestamp_test.cc b/mediapipe/framework/timestamp_test.cc index 5f5cc342..3ba0b5c3 100644 --- a/mediapipe/framework/timestamp_test.cc +++ b/mediapipe/framework/timestamp_test.cc @@ -125,6 +125,22 @@ TEST(TimestampTest, NextAllowedInStream) { Timestamp::PostStream().NextAllowedInStream()); } +TEST(TimestampTest, HasNextAllowedInStream) { + EXPECT_TRUE(Timestamp::Min().HasNextAllowedInStream()); + EXPECT_TRUE((Timestamp::Min() + 1).HasNextAllowedInStream()); + EXPECT_TRUE(Timestamp(-1000).HasNextAllowedInStream()); + EXPECT_TRUE(Timestamp(0).HasNextAllowedInStream()); + EXPECT_TRUE(Timestamp(1000).HasNextAllowedInStream()); + EXPECT_TRUE((Timestamp::Max() - 2).HasNextAllowedInStream()); + EXPECT_TRUE((Timestamp::Max() - 1).HasNextAllowedInStream()); + + EXPECT_FALSE(Timestamp::PreStream().HasNextAllowedInStream()); + EXPECT_FALSE(Timestamp::Max().HasNextAllowedInStream()); + EXPECT_FALSE(Timestamp::PostStream().HasNextAllowedInStream()); + EXPECT_FALSE(Timestamp::OneOverPostStream().HasNextAllowedInStream()); + EXPECT_FALSE(Timestamp::Done().HasNextAllowedInStream()); +} + TEST(TimestampTest, SpecialValueDifferences) { { // Lower range const std::vector timestamps = { From b19b80e10f021978b01706d9d7530b6e19fd860e Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 13 Jun 2023 01:50:58 -0700 Subject: [PATCH 4/9] Add support for int64 constant side package value. PiperOrigin-RevId: 539893314 --- .../calculators/core/constant_side_packet_calculator.cc | 4 ++++ .../calculators/core/constant_side_packet_calculator.proto | 5 +++-- .../calculators/core/constant_side_packet_calculator_test.cc | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mediapipe/calculators/core/constant_side_packet_calculator.cc b/mediapipe/calculators/core/constant_side_packet_calculator.cc index 509f7e9d..0bcf22ec 100644 --- a/mediapipe/calculators/core/constant_side_packet_calculator.cc +++ b/mediapipe/calculators/core/constant_side_packet_calculator.cc @@ -87,6 +87,8 @@ class ConstantSidePacketCalculator : public CalculatorBase { packet.Set(); } else if (packet_options.has_time_series_header_value()) { packet.Set(); + } else if (packet_options.has_int64_value()) { + packet.Set(); } else { return absl::InvalidArgumentError( "None of supported values were specified in options."); @@ -124,6 +126,8 @@ class ConstantSidePacketCalculator : public CalculatorBase { } else if (packet_options.has_time_series_header_value()) { packet.Set(MakePacket( packet_options.time_series_header_value())); + } else if (packet_options.has_int64_value()) { + packet.Set(MakePacket(packet_options.int64_value())); } else { return absl::InvalidArgumentError( "None of supported values were specified in options."); diff --git a/mediapipe/calculators/core/constant_side_packet_calculator.proto b/mediapipe/calculators/core/constant_side_packet_calculator.proto index 78a773a6..bce82705 100644 --- a/mediapipe/calculators/core/constant_side_packet_calculator.proto +++ b/mediapipe/calculators/core/constant_side_packet_calculator.proto @@ -29,13 +29,14 @@ message ConstantSidePacketCalculatorOptions { message ConstantSidePacket { oneof value { int32 int_value = 1; + uint64 uint64_value = 5; + int64 int64_value = 11; float float_value = 2; + double double_value = 9; bool bool_value = 3; string string_value = 4; - uint64 uint64_value = 5; ClassificationList classification_list_value = 6; LandmarkList landmark_list_value = 7; - double double_value = 9; TimeSeriesHeader time_series_header_value = 10; } } diff --git a/mediapipe/calculators/core/constant_side_packet_calculator_test.cc b/mediapipe/calculators/core/constant_side_packet_calculator_test.cc index a7ff808f..6e8c0ec3 100644 --- a/mediapipe/calculators/core/constant_side_packet_calculator_test.cc +++ b/mediapipe/calculators/core/constant_side_packet_calculator_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include "absl/strings/string_view.h" @@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) { DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f); DoTestSingleSidePacket("{ bool_value: true }", true); DoTestSingleSidePacket(R"({ string_value: "str" })", "str"); + DoTestSingleSidePacket("{ int64_value: 63 }", 63); } TEST(ConstantSidePacketCalculatorTest, MultiplePackets) { From e468bee58419e741d1ab4b4b2d4c47d41fa911ae Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 13 Jun 2023 09:51:42 -0700 Subject: [PATCH 5/9] Deprecate GraphStatus() PiperOrigin-RevId: 539992850 --- mediapipe/framework/calculator_context.h | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/mediapipe/framework/calculator_context.h b/mediapipe/framework/calculator_context.h index 284226d9..9568ba74 100644 --- a/mediapipe/framework/calculator_context.h +++ b/mediapipe/framework/calculator_context.h @@ -109,9 +109,20 @@ class CalculatorContext { // use OutputStream::SetOffset() directly. void SetOffset(TimestampDiff offset); - // Returns the status of the graph run. + // DEPRECATED: This was intended to get graph run status during + // `CalculatorBase::Close` call. However, `Close` can run simultaneously with + // other calculators `CalculatorBase::Process`, hence the actual graph + // status may change any time and returned graph status here does not + // necessarily reflect the actual graph status. // - // NOTE: This method should only be called during CalculatorBase::Close(). + // As an alternative, instead of checking graph status in `Close` and doing + // work for "done" state, you can enable timestamp bound processing for your + // calculator (`CalculatorContract::SetProcessTimestampBounds`) to trigger + // `Process` on timestamp bound updates and handle "done" state there. + // Check examples in: + // mediapipe/framework/calculator_graph_summary_packet_test.cc. + // + ABSL_DEPRECATED("Does not reflect the actual graph status.") absl::Status GraphStatus() const { return graph_status_; } ProfilingContext* GetProfilingContext() const { From b97d11fa76c88cb15b2e336b4c827821a09783c2 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 13 Jun 2023 15:00:40 -0700 Subject: [PATCH 6/9] Internal MediaPipe Tasks change PiperOrigin-RevId: 540083633 --- mediapipe/calculators/tensor/BUILD | 2 -- .../tensor/bert_preprocessor_calculator.cc | 4 ++-- .../tensor/inference_interpreter_delegate_runner.cc | 13 +++++++++++++ .../tensor/regex_preprocessor_calculator.cc | 3 +-- mediapipe/framework/formats/tensor.h | 7 +++++++ mediapipe/framework/formats/tensor_test.cc | 12 ++++++++++++ 6 files changed, 35 insertions(+), 6 deletions(-) diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 2ad98f28..a3e61c06 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -228,7 +228,6 @@ cc_library( "//mediapipe/tasks/metadata:metadata_schema_cc", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -280,7 +279,6 @@ cc_library( "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils", "//mediapipe/tasks/metadata:metadata_schema_cc", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", ], alwayslink = 1, ) diff --git a/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc b/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc index b5612280..12db1493 100644 --- a/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc +++ b/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc @@ -22,7 +22,6 @@ #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "absl/strings/ascii.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" @@ -244,7 +243,8 @@ std::vector BertPreprocessorCalculator::GenerateInputTensors( input_tensors.reserve(kNumInputTensorsForBert); for (int i = 0; i < kNumInputTensorsForBert; ++i) { input_tensors.push_back( - {Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})}); + {Tensor::ElementType::kInt32, + Tensor::Shape({1, tensor_size}, has_dynamic_input_tensors_)}); } std::memcpy(input_tensors[input_ids_tensor_index_] .GetCpuWriteView() diff --git a/mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc b/mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc index a2b8a928..b727f179 100644 --- a/mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc +++ b/mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc @@ -96,6 +96,19 @@ absl::StatusOr> InferenceInterpreterDelegateRunner::Run( CalculatorContext* cc, const std::vector& input_tensors) { // Read CPU input into tensors. RET_CHECK_EQ(interpreter_->inputs().size(), input_tensors.size()); + + // If the input tensors have dynamic shape, then the tensors need to be + // resized and reallocated before we can copy the tensor values. + bool resized_tensor_shapes = false; + for (int i = 0; i < input_tensors.size(); ++i) { + if (input_tensors[i].shape().is_dynamic) { + interpreter_->ResizeInputTensorStrict(i, input_tensors[i].shape().dims); + resized_tensor_shapes = true; + } + } + // Reallocation is needed for memory sanity. + if (resized_tensor_shapes) interpreter_->AllocateTensors(); + for (int i = 0; i < input_tensors.size(); ++i) { const TfLiteType input_tensor_type = interpreter_->tensor(interpreter_->inputs()[i])->type; diff --git a/mediapipe/calculators/tensor/regex_preprocessor_calculator.cc b/mediapipe/calculators/tensor/regex_preprocessor_calculator.cc index 92a5f026..8276462f 100644 --- a/mediapipe/calculators/tensor/regex_preprocessor_calculator.cc +++ b/mediapipe/calculators/tensor/regex_preprocessor_calculator.cc @@ -20,7 +20,6 @@ #include #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "mediapipe/calculators/tensor/regex_preprocessor_calculator.pb.h" #include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/api2/port.h" @@ -161,7 +160,7 @@ absl::Status RegexPreprocessorCalculator::Process(CalculatorContext* cc) { // not found in the tokenizer vocab. std::vector result; result.push_back( - {Tensor::ElementType::kInt32, Tensor::Shape({max_seq_len_})}); + {Tensor::ElementType::kInt32, Tensor::Shape({1, max_seq_len_})}); std::memcpy(result[0].GetCpuWriteView().buffer(), input_tokens.data(), input_tokens.size() * sizeof(int32_t)); kTensorsOut(cc).Send(std::move(result)); diff --git a/mediapipe/framework/formats/tensor.h b/mediapipe/framework/formats/tensor.h index 1d670d80..4f95eb27 100644 --- a/mediapipe/framework/formats/tensor.h +++ b/mediapipe/framework/formats/tensor.h @@ -117,11 +117,18 @@ class Tensor { Shape() = default; Shape(std::initializer_list dimensions) : dims(dimensions) {} Shape(const std::vector& dimensions) : dims(dimensions) {} + Shape(std::initializer_list dimensions, bool is_dynamic) + : dims(dimensions), is_dynamic(is_dynamic) {} + Shape(const std::vector& dimensions, bool is_dynamic) + : dims(dimensions), is_dynamic(is_dynamic) {} int num_elements() const { return std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); } std::vector dims; + // The Tensor has dynamic rather than static shape so the TFLite interpreter + // needs to be reallocated. Only relevant for CPU. + bool is_dynamic = false; }; // Quantization parameters corresponding to the zero_point and scale value // made available by TfLite quantized (uint8/int8) tensors. diff --git a/mediapipe/framework/formats/tensor_test.cc b/mediapipe/framework/formats/tensor_test.cc index 4ad4e18e..468af4ab 100644 --- a/mediapipe/framework/formats/tensor_test.cc +++ b/mediapipe/framework/formats/tensor_test.cc @@ -2,6 +2,7 @@ #include #include +#include #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" @@ -34,6 +35,17 @@ TEST(General, TestDataTypes) { EXPECT_EQ(t_bool.bytes(), t_bool.shape().num_elements() * sizeof(bool)); } +TEST(General, TestDynamic) { + Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape({1, 2, 3, 4}, true)); + EXPECT_EQ(t1.shape().num_elements(), 1 * 2 * 3 * 4); + EXPECT_TRUE(t1.shape().is_dynamic); + + std::vector t2_dims = {4, 3, 2, 3}; + Tensor t2(Tensor::ElementType::kFloat16, Tensor::Shape(t2_dims, true)); + EXPECT_EQ(t2.shape().num_elements(), 4 * 3 * 2 * 3); + EXPECT_TRUE(t2.shape().is_dynamic); +} + TEST(Cpu, TestMemoryAllocation) { Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape{4, 3, 2, 3}); auto v1 = t1.GetCpuWriteView(); From 02d55dfb0a2a4475effb4d4d3dbd280ee4a5dc0f Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 13 Jun 2023 16:18:27 -0700 Subject: [PATCH 7/9] Modify the TensorToImageFrameCalculator to support normalized outputs. PiperOrigin-RevId: 540104988 --- mediapipe/calculators/tensorflow/BUILD | 1 + .../tensor_to_image_frame_calculator.cc | 31 ++++++++++-- .../tensor_to_image_frame_calculator.proto | 4 ++ .../tensor_to_image_frame_calculator_test.cc | 50 ++++++++++++++++++- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index c4b9ab9f..feee2372 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -1077,6 +1077,7 @@ cc_test( linkstatic = 1, deps = [ ":tensor_to_image_frame_calculator", + ":tensor_to_image_frame_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/formats:image_frame", diff --git a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc index 34e397b3..b5a94e01 100644 --- a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc @@ -65,6 +65,7 @@ class TensorToImageFrameCalculator : public CalculatorBase { private: float scale_factor_; + bool scale_per_frame_min_max_; }; REGISTER_CALCULATOR(TensorToImageFrameCalculator); @@ -88,6 +89,8 @@ absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) { absl::Status TensorToImageFrameCalculator::Open(CalculatorContext* cc) { scale_factor_ = cc->Options().scale_factor(); + scale_per_frame_min_max_ = cc->Options() + .scale_per_frame_min_max(); cc->SetOffset(TimestampDiff(0)); return absl::OkStatus(); } @@ -109,16 +112,38 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) { auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8); const int32_t total_size = height * width * depth; + if (scale_per_frame_min_max_) { + RET_CHECK_EQ(input_tensor.dtype(), tensorflow::DT_FLOAT) + << "Setting scale_per_frame_min_max requires FLOAT input tensors."; + } ::std::unique_ptr output; if (input_tensor.dtype() == tensorflow::DT_FLOAT) { // Allocate buffer with alignments. std::unique_ptr buffer( new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]); auto data = input_tensor.flat().data(); + float min = 1e23; + float max = -1e23; + if (scale_per_frame_min_max_) { + for (int i = 0; i < total_size; ++i) { + float d = scale_factor_ * data[i]; + if (d < min) { + min = d; + } + if (d > max) { + max = d; + } + } + } for (int i = 0; i < total_size; ++i) { - float d = scale_factor_ * data[i]; - if (d < 0) d = 0; - if (d > 255) d = 255; + float d = data[i]; + if (scale_per_frame_min_max_) { + d = 255 * (d - min) / (max - min + 1e-9); + } else { + d = scale_factor_ * d; + if (d < 0) d = 0; + if (d > 255) d = 255; + } buffer[i] = d; } output = ::absl::make_unique( diff --git a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto index 3410068d..c60448c1 100644 --- a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto +++ b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto @@ -26,4 +26,8 @@ message TensorToImageFrameCalculatorOptions { // Multiples floating point tensor outputs by this value before converting to // uint8. This is useful for converting from range [0, 1] to [0, 255] optional float scale_factor = 1 [default = 1.0]; + + // If true, scales any FLOAT tensor input of [min, max] to be between [0, 255] + // per frame. This overrides any explicit scale_factor. + optional bool scale_per_frame_min_max = 2 [default = false]; } diff --git a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc index aee9fee9..13255ac4 100644 --- a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc +++ b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc @@ -11,7 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include +#include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/formats/image_frame.h" @@ -32,11 +34,14 @@ constexpr char kImage[] = "IMAGE"; template class TensorToImageFrameCalculatorTest : public ::testing::Test { protected: - void SetUpRunner() { + void SetUpRunner(bool scale_per_frame_min_max = false) { CalculatorGraphConfig::Node config; config.set_calculator("TensorToImageFrameCalculator"); config.add_input_stream("TENSOR:input_tensor"); config.add_output_stream("IMAGE:output_image"); + config.mutable_options() + ->MutableExtension(mediapipe::TensorToImageFrameCalculatorOptions::ext) + ->set_scale_per_frame_min_max(scale_per_frame_min_max); runner_ = absl::make_unique(config); } @@ -157,4 +162,47 @@ TYPED_TEST(TensorToImageFrameCalculatorTest, } } +TYPED_TEST(TensorToImageFrameCalculatorTest, + Converts3DTensorToImageFrame2DGrayWithScaling) { + this->SetUpRunner(true); + auto& runner = this->runner_; + constexpr int kWidth = 16; + constexpr int kHeight = 8; + const tf::TensorShape tensor_shape{kHeight, kWidth}; + auto tensor = absl::make_unique( + tf::DataTypeToEnum::v(), tensor_shape); + auto tensor_vec = tensor->template flat().data(); + + // Writing sequence of integers as floats which we want normalized. + tensor_vec[0] = 255; + for (int i = 1; i < kWidth * kHeight; ++i) { + tensor_vec[i] = 200; + } + + const int64_t time = 1234; + runner->MutableInputs()->Tag(kTensor).packets.push_back( + Adopt(tensor.release()).At(Timestamp(time))); + + if (!std::is_same::value) { + EXPECT_FALSE(runner->Run().ok()); + return; // Short circuit because does not apply to other types. + } else { + EXPECT_TRUE(runner->Run().ok()); + const std::vector& output_packets = + runner->Outputs().Tag(kImage).packets; + EXPECT_EQ(1, output_packets.size()); + EXPECT_EQ(time, output_packets[0].Timestamp().Value()); + const ImageFrame& output_image = output_packets[0].Get(); + EXPECT_EQ(ImageFormat::GRAY8, output_image.Format()); + EXPECT_EQ(kWidth, output_image.Width()); + EXPECT_EQ(kHeight, output_image.Height()); + + EXPECT_EQ(255, output_image.PixelData()[0]); + for (int i = 1; i < kWidth * kHeight; ++i) { + const uint8_t pixel_value = output_image.PixelData()[i]; + ASSERT_EQ(0, pixel_value); + } + } +} + } // namespace mediapipe From 3742bc8c1b9f1325c4588f029c826495d109e25c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 13 Jun 2023 17:10:11 -0700 Subject: [PATCH 8/9] Add metadata for all PREFIX/image... prefixes. PiperOrigin-RevId: 540117214 --- mediapipe/util/sequence/media_sequence.cc | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mediapipe/util/sequence/media_sequence.cc b/mediapipe/util/sequence/media_sequence.cc index 287db618..21d030ff 100644 --- a/mediapipe/util/sequence/media_sequence.cc +++ b/mediapipe/util/sequence/media_sequence.cc @@ -147,6 +147,22 @@ absl::Status ReconcileMetadataImages(const std::string& prefix, return absl::OkStatus(); } +// Reconciles metadata for all images. +absl::Status ReconcileMetadataImages(tensorflow::SequenceExample* sequence) { + RET_CHECK_OK(ReconcileMetadataImages("", sequence)); + for (const auto& key_value : sequence->feature_lists().feature_list()) { + const auto& key = key_value.first; + if (::absl::StrContains(key, kImageTimestampKey)) { + std::string prefix = ""; + if (key != kImageTimestampKey) { + prefix = key.substr(0, key.size() - sizeof(kImageTimestampKey)); + } + RET_CHECK_OK(ReconcileMetadataImages(prefix, sequence)); + } + } + return absl::OkStatus(); +} + // Sets the values of "feature/${TAG}/dimensions", and // "feature/${TAG}/frame_rate" for each float list feature TAG. If the // dimensions are already present as a context feature, this method verifies @@ -545,10 +561,7 @@ absl::Status ReconcileMetadata(bool reconcile_bbox_annotations, bool reconcile_region_annotations, tensorflow::SequenceExample* sequence) { RET_CHECK_OK(ReconcileAnnotationIndicesByImageTimestamps(sequence)); - RET_CHECK_OK(ReconcileMetadataImages("", sequence)); - RET_CHECK_OK(ReconcileMetadataImages(kForwardFlowPrefix, sequence)); - RET_CHECK_OK(ReconcileMetadataImages(kClassSegmentationPrefix, sequence)); - RET_CHECK_OK(ReconcileMetadataImages(kInstanceSegmentationPrefix, sequence)); + RET_CHECK_OK(ReconcileMetadataImages(sequence)); RET_CHECK_OK(ReconcileMetadataFeatureFloats(sequence)); if (reconcile_bbox_annotations) { RET_CHECK_OK(ReconcileMetadataBoxAnnotations("", sequence)); From eaeca82b76bdaca89c2411cbec97dd3a346e3c44 Mon Sep 17 00:00:00 2001 From: Yuqi Li Date: Tue, 13 Jun 2023 18:39:14 -0700 Subject: [PATCH 9/9] Internal change PiperOrigin-RevId: 540134258 --- mediapipe/util/cpu_util.cc | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/mediapipe/util/cpu_util.cc b/mediapipe/util/cpu_util.cc index 052eabb8..74e6debd 100644 --- a/mediapipe/util/cpu_util.cc +++ b/mediapipe/util/cpu_util.cc @@ -26,7 +26,6 @@ #include #include "absl/algorithm/container.h" -#include "absl/flags/flag.h" #include "absl/strings/match.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" @@ -35,23 +34,14 @@ #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/statusor.h" -ABSL_FLAG(std::string, system_cpu_max_freq_file, - "/sys/devices/system/cpu/cpu$0/cpufreq/cpuinfo_max_freq", - "The file pattern for CPU max frequencies, where $0 will be replaced " - "with the CPU id."); - namespace mediapipe { namespace { constexpr uint32_t kBufferLength = 64; absl::StatusOr GetFilePath(int cpu) { - if (!absl::StrContains(absl::GetFlag(FLAGS_system_cpu_max_freq_file), "$0")) { - return absl::InvalidArgumentError( - absl::StrCat("Invalid frequency file: ", - absl::GetFlag(FLAGS_system_cpu_max_freq_file))); - } - return absl::Substitute(absl::GetFlag(FLAGS_system_cpu_max_freq_file), cpu); + return absl::Substitute( + "/sys/devices/system/cpu/cpu$0/cpufreq/cpuinfo_max_freq", cpu); } absl::StatusOr GetCpuMaxFrequency(int cpu) {